diff options
362 files changed, 5556 insertions, 16461 deletions
diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index c87e9855fe9..07361e4b2eb 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -13,3 +13,4 @@ add_custom_target(vespalog_logfmt ALL DEPENDS ${GODIR}/bin/vespa-logfmt) install(PROGRAMS ${GODIR}/bin/vespa-logfmt DESTINATION bin) install(PROGRAMS ${GODIR}/bin/vespa-deploy DESTINATION bin) +install(PROGRAMS ${GODIR}/bin/script-utils DESTINATION libexec/vespa) diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index ac4d5085782..3b16ec4d342 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -100,11 +100,15 @@ func doCert(cli *CLI, overwriteCertificate, noApplicationPackage bool, args []st return err } } - privateKeyFile, err := cli.config.privateKeyPath(app) + targetType, err := cli.config.targetType() if err != nil { return err } - certificateFile, err := cli.config.certificatePath(app) + privateKeyFile, err := cli.config.privateKeyPath(app, targetType) + if err != nil { + return err + } + certificateFile, err := cli.config.certificatePath(app, targetType) if err != nil { return err } @@ -167,7 +171,11 @@ func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { if err != nil { return err } - certificateFile, err := cli.config.certificatePath(app) + targetType, err := cli.config.targetType() + if err != nil { + return err + } + certificateFile, err := cli.config.certificatePath(app, targetType) if err != nil { return err } diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 8f81f8e359f..acca161727b 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -6,6 +6,7 @@ package cmd import ( "crypto/tls" + "crypto/x509" "fmt" "log" "os" @@ -298,6 +299,14 @@ func loadConfigFrom(dir string, environment map[string]string, flags map[string] return c, nil } +func athenzPath(filename string) (string, error) { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(userHome, ".athenz", filename), nil +} + func (c *Config) loadLocalConfigFrom(parent string) error { home := filepath.Join(parent, ".vespa") _, err := os.Stat(home) @@ -383,44 +392,69 @@ func (c *Config) deploymentIn(system vespa.System) (vespa.Deployment, error) { return vespa.Deployment{System: system, Application: app, Zone: zone}, nil } -func (c *Config) certificatePath(app vespa.ApplicationID) (string, error) { +func (c *Config) certificatePath(app vespa.ApplicationID, targetType string) (string, error) { if override, ok := c.environment["VESPA_CLI_DATA_PLANE_CERT_FILE"]; ok { return override, nil } + if targetType == vespa.TargetHosted { + return athenzPath("cert") + } return c.applicationFilePath(app, "data-plane-public-cert.pem") } -func (c *Config) privateKeyPath(app vespa.ApplicationID) (string, error) { +func (c *Config) privateKeyPath(app vespa.ApplicationID, targetType string) (string, error) { if override, ok := c.environment["VESPA_CLI_DATA_PLANE_KEY_FILE"]; ok { return override, nil } + if targetType == vespa.TargetHosted { + return athenzPath("key") + } return c.applicationFilePath(app, "data-plane-private-key.pem") } -func (c *Config) x509KeyPair(app vespa.ApplicationID) (KeyPair, error) { +func (c *Config) x509KeyPair(app vespa.ApplicationID, targetType string) (KeyPair, error) { cert, certOk := c.environment["VESPA_CLI_DATA_PLANE_CERT"] key, keyOk := c.environment["VESPA_CLI_DATA_PLANE_KEY"] + var ( + kp tls.Certificate + err error + certFile string + keyFile string + ) if certOk && keyOk { // Use key pair from environment - kp, err := tls.X509KeyPair([]byte(cert), []byte(key)) - return KeyPair{KeyPair: kp}, err - } - privateKeyFile, err := c.privateKeyPath(app) - if err != nil { - return KeyPair{}, err + kp, err = tls.X509KeyPair([]byte(cert), []byte(key)) + } else { + keyFile, err = c.privateKeyPath(app, targetType) + if err != nil { + return KeyPair{}, err + } + certFile, err = c.certificatePath(app, targetType) + if err != nil { + return KeyPair{}, err + } + kp, err = tls.LoadX509KeyPair(certFile, keyFile) } - certificateFile, err := c.certificatePath(app) if err != nil { return KeyPair{}, err } - kp, err := tls.LoadX509KeyPair(certificateFile, privateKeyFile) - if err != nil { - return KeyPair{}, err + if targetType == vespa.TargetHosted { + cert, err := x509.ParseCertificate(kp.Certificate[0]) + if err != nil { + return KeyPair{}, err + } + now := time.Now() + expiredAt := cert.NotAfter + if expiredAt.Before(now) { + delta := now.Sub(expiredAt).Truncate(time.Second) + return KeyPair{}, fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta) + } + return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil } return KeyPair{ KeyPair: kp, - CertificateFile: certificateFile, - PrivateKeyFile: privateKeyFile, + CertificateFile: certFile, + PrivateKeyFile: keyFile, }, nil } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 012570c5471..df08c90768b 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -78,13 +78,13 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, } log.Println() - if opts.IsCloud() { + if opts.Target.IsCloud() { cli.printSuccess("Triggered deployment of ", color.CyanString(pkg.Path), " with run ID ", color.CyanString(strconv.FormatInt(result.ID, 10))) } else { cli.printSuccess("Deployed ", color.CyanString(pkg.Path)) printPrepareLog(cli.Stderr, result) } - if opts.IsCloud() { + if opts.Target.IsCloud() { log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status")) log.Print(color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/%s/instance/%s/job/%s-%s/run/%d", opts.Target.Deployment().System.ConsoleURL, diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 9ccc39e02a1..3c1799701a3 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -162,12 +162,12 @@ func TestProdSubmit(t *testing.T) { assert.Nil(t, cli.Run("auth", "cert", pkgDir)) // Remove certificate as it's not required for submission (but it must be part of the application package) - if path, err := cli.config.privateKeyPath(app); err == nil { + if path, err := cli.config.privateKeyPath(app, vespa.TargetCloud); err == nil { os.RemoveAll(path) } else { require.Nil(t, err) } - if path, err := cli.config.certificatePath(app); err == nil { + if path, err := cli.config.certificatePath(app, vespa.TargetCloud); err == nil { os.RemoveAll(path) } else { require.Nil(t, err) diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index e2f03cbc7ce..d41be3fa097 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -2,15 +2,12 @@ package cmd import ( - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io" "log" "os" "os/exec" - "path/filepath" "strings" "time" @@ -332,7 +329,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta authConfigPath = c.config.authConfigPath() deploymentTLSOptions = vespa.TLSOptions{} if !opts.noCertificate { - kp, err := c.config.x509KeyPair(deployment.Application) + kp, err := c.config.x509KeyPair(deployment.Application, targetType) if err != nil { return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") } @@ -343,9 +340,9 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta } } case vespa.TargetHosted: - kp, err := athenzKeyPair() + kp, err := c.config.x509KeyPair(deployment.Application, targetType) if err != nil { - return nil, err + return nil, errHint(err, "Deployment to hosted requires an Athenz certificate", "Try renewing certificate with 'athenz-user-cert'") } apiTLSOptions = vespa.TLSOptions{ KeyPair: kp.KeyPair, @@ -406,7 +403,11 @@ func (c *CLI) service(target vespa.Target, name string, sessionOrRunID int64, cl } s, err := target.Service(name, timeout, sessionOrRunID, cluster) if err != nil { - return nil, fmt.Errorf("service '%s' is unavailable: %w", name, err) + err := fmt.Errorf("service '%s' is unavailable: %w", name, err) + if target.IsCloud() { + return nil, errHint(err, "Confirm that you're communicating with the correct zone and cluster", "The -z option controls the zone", "The -C option controls the cluster") + } + return nil, err } return s, nil } @@ -487,40 +488,6 @@ func isTerminal(w io.Writer) bool { return false } -func athenzPath(filename string) (string, error) { - userHome, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(userHome, ".athenz", filename), nil -} - -func athenzKeyPair() (KeyPair, error) { - certFile, err := athenzPath("cert") - if err != nil { - return KeyPair{}, err - } - keyFile, err := athenzPath("key") - if err != nil { - return KeyPair{}, err - } - kp, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return KeyPair{}, err - } - cert, err := x509.ParseCertificate(kp.Certificate[0]) - if err != nil { - return KeyPair{}, err - } - now := time.Now() - expiredAt := cert.NotAfter - if expiredAt.Before(now) { - delta := now.Sub(expiredAt).Truncate(time.Second) - return KeyPair{}, errHint(fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta), "Try renewing certificate with 'athenz-user-cert'") - } - return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil -} - // applicationPackageFrom returns an application loaded from args. If args is empty, the application package is loaded // from the working directory. If requirePackaging is true, the application package is required to be packaged with mvn // package. diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index c44bbddb98a..56f394d94ee 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -74,25 +74,27 @@ func printServiceStatus(cli *CLI, name string) error { if err != nil { return err } - timeout, err := cli.config.timeout() + cluster := cli.config.cluster() + s, err := cli.service(t, name, 0, cluster) if err != nil { return err } - if timeout > 0 { - log.Printf("Waiting up to %s for service to become ready ...", color.CyanString(timeout.String())) - } - s, err := t.Service(name, timeout, 0, cli.config.cluster()) + timeout, err := cli.config.timeout() if err != nil { return err } status, err := s.Wait(timeout) + clusterPart := "" + if cluster != "" { + clusterPart = fmt.Sprintf(" named %s", color.CyanString(cluster)) + } if status/100 == 2 { - log.Print(s.Description(), " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) + log.Print(s.Description(), clusterPart, " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) } else { if err == nil { err = fmt.Errorf("status %d", status) } - return fmt.Errorf("%s at %s is %s: %w", s.Description(), color.CyanString(s.BaseURL), color.RedString("not ready"), err) + return fmt.Errorf("%s%s at %s is %s: %w", s.Description(), clusterPart, color.CyanString(s.BaseURL), color.RedString("not ready"), err) } return nil } diff --git a/client/go/script-utils/main.go b/client/go/script-utils/main.go new file mode 100644 index 00000000000..a7160691a5d --- /dev/null +++ b/client/go/script-utils/main.go @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package main + +import ( + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/vespa" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "actions: export-env, ipv6-only") + return + } + switch os.Args[1] { + case "export-env": + vespa.ExportDefaultEnvToSh() + case "ipv6-only": + if vespa.HasOnlyIpV6() { + os.Exit(0) + } else { + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "unknown action '%s'\n", os.Args[1]) + } +} diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index 0e086979d72..b98679aadd8 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -80,11 +80,6 @@ func (d DeploymentOptions) String() string { return fmt.Sprintf("%s to %s", d.Target.Deployment(), d.Target.Type()) } -// IsCloud returns whether this is a deployment to Vespa Cloud or hosted Vespa -func (d *DeploymentOptions) IsCloud() bool { - return d.Target.Type() == TargetCloud || d.Target.Type() == TargetHosted -} - func (d *DeploymentOptions) url(path string) (*url.URL, error) { service, err := d.Target.Service(DeployService, 0, 0, "") if err != nil { @@ -115,7 +110,7 @@ func ZoneFromString(s string) (ZoneID, error) { // Prepare deployment and return the session ID func Prepare(deployment DeploymentOptions) (PrepareResult, error) { - if deployment.IsCloud() { + if deployment.Target.IsCloud() { return PrepareResult{}, fmt.Errorf("prepare is not supported with %s target", deployment.Target.Type()) } sessionURL, err := deployment.url("/application/v2/tenant/default/session") @@ -164,7 +159,7 @@ func Prepare(deployment DeploymentOptions) (PrepareResult, error) { // Activate deployment with sessionID from a past prepare func Activate(sessionID int64, deployment DeploymentOptions) error { - if deployment.IsCloud() { + if deployment.Target.IsCloud() { return fmt.Errorf("activate is not supported with %s target", deployment.Target.Type()) } u, err := deployment.url(fmt.Sprintf("/application/v2/tenant/default/session/%d/active", sessionID)) @@ -186,7 +181,7 @@ func Activate(sessionID int64, deployment DeploymentOptions) error { func Deploy(opts DeploymentOptions) (PrepareResult, error) { path := "/application/v2/tenant/default/prepareandactivate" - if opts.IsCloud() { + if opts.Target.IsCloud() { if err := checkDeploymentOpts(opts); err != nil { return PrepareResult{}, err } @@ -225,7 +220,7 @@ func copyToPart(dst *multipart.Writer, src io.Reader, fieldname, filename string } func Submit(opts DeploymentOptions) error { - if !opts.IsCloud() { + if !opts.Target.IsCloud() { return fmt.Errorf("%s: submit is unsupported by %s target", opts, opts.Target.Type()) } if err := checkDeploymentOpts(opts); err != nil { @@ -282,7 +277,7 @@ func checkDeploymentOpts(opts DeploymentOptions) error { if opts.Target.Type() == TargetCloud && !opts.ApplicationPackage.HasCertificate() { return fmt.Errorf("%s: missing certificate in package", opts) } - if !opts.IsCloud() && !opts.Version.IsZero() { + if !opts.Target.IsCloud() && !opts.Version.IsZero() { return fmt.Errorf("%s: custom runtime version is not supported by %s target", opts, opts.Target.Type()) } return nil @@ -295,7 +290,7 @@ func newDeploymentRequest(url *url.URL, opts DeploymentOptions) (*http.Request, } var body io.Reader header := http.Header{} - if opts.IsCloud() { + if opts.Target.IsCloud() { var buf bytes.Buffer form := multipart.NewWriter(&buf) formFile, err := form.CreateFormFile("applicationZip", filepath.Base(opts.ApplicationPackage.Path)) diff --git a/client/go/vespa/detect_hostname.go b/client/go/vespa/detect_hostname.go new file mode 100644 index 00000000000..d4d34a5f47d --- /dev/null +++ b/client/go/vespa/detect_hostname.go @@ -0,0 +1,137 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Author: arnej + +package vespa + +import ( + "fmt" + "net" + "os" + "strings" +) + +type lookupAddrFunc func(addr string) ([]string, error) +type lookupIPFunc func(host string) ([]net.IP, error) + +// detect if this host is IPv6-only, in which case we want to pass +// the flag "-Djava.net.preferIPv6Addresses=true" to any java command +func HasOnlyIpV6() bool { + hostname, err := FindOurHostname() + if hostname == "" || err != nil { + return false + } + foundV4 := false + foundV6 := false + ipAddrs, err := net.LookupIP(hostname) + if err != nil { + return false + } + for _, addr := range ipAddrs { + switch { + case addr.IsLoopback(): + // skip + case addr.To4() != nil: + foundV4 = true + case addr.To16() != nil: + foundV6 = true + } + } + return foundV6 && !foundV4 +} + +// Find a good name for the host we're running on. +// We need something that *other* hosts can use for connnecting back +// to our services, preferably the canonical DNS name. +// If automatic detection fails, "localhost" will be returned, so +// single-node setups still have a good chance of working. +// Use the enviroment variable VESPA_HOSTNAME to override. +func FindOurHostname() (string, error) { return findOurHostname(net.LookupAddr, net.LookupIP) } + +func findOurHostname(lookupAddr lookupAddrFunc, lookupIP lookupIPFunc) (string, error) { + env := os.Getenv("VESPA_HOSTNAME") + if env != "" { + // assumes: env var is already validated and OK + return env, nil + } + name, err := os.Hostname() + if err != nil { + return findOurHostnameFrom("localhost", lookupAddr, lookupIP) + } + name, err = findOurHostnameFrom(name, lookupAddr, lookupIP) + return strings.TrimSuffix(name, "."), err +} + +func validateHostname(name string) bool { + myIpAddresses := make(map[string]bool) + interfaceAddrs, _ := net.InterfaceAddrs() + for _, ifAddr := range interfaceAddrs { + // note: ifAddr.String() is typically "127.0.0.1/8" + if ipnet, ok := ifAddr.(*net.IPNet); ok { + myIpAddresses[ipnet.IP.String()] = true + } + } + ipAddrs, _ := net.LookupIP(name) + someGood := false + for _, addr := range ipAddrs { + if len(myIpAddresses) == 0 { + // no validation possible, assume OK + return true + } + if myIpAddresses[addr.String()] { + someGood = true + } else { + return false + } + } + return someGood +} + +func findOurHostnameFrom(name string, lookupAddr lookupAddrFunc, lookupIP lookupIPFunc) (string, error) { + if strings.Contains(name, ".") && validateHostname(name) { + // it's all good + return name, nil + } + possibles := make([]string, 0, 5) + if name != "" { + ipAddrs, _ := lookupIP(name) + for _, addr := range ipAddrs { + switch { + case addr.IsLoopback(): + // skip + case addr.To4() != nil || addr.To16() != nil: + reverseNames, _ := lookupAddr(addr.String()) + possibles = append(possibles, reverseNames...) + } + } + } + interfaceAddrs, _ := net.InterfaceAddrs() + for _, ifAddr := range interfaceAddrs { + if ipnet, ok := ifAddr.(*net.IPNet); ok { + ip := ipnet.IP + if ip == nil || ip.IsLoopback() { + continue + } + reverseNames, _ := lookupAddr(ip.String()) + possibles = append(possibles, reverseNames...) + } + } + // look for valid possible starting with the given name + for _, poss := range possibles { + if strings.HasPrefix(poss, name+".") && validateHostname(poss) { + return poss, nil + } + } + // look for valid possible + for _, poss := range possibles { + if strings.Contains(poss, ".") && validateHostname(poss) { + return poss, nil + } + } + // look for any valid possible + for _, poss := range possibles { + if validateHostname(poss) { + return poss, nil + } + } + return "localhost", fmt.Errorf("fallback to localhost, os.Hostname '%s'", name) +} diff --git a/client/go/vespa/detect_hostname_test.go b/client/go/vespa/detect_hostname_test.go new file mode 100644 index 00000000000..e162bcdea8e --- /dev/null +++ b/client/go/vespa/detect_hostname_test.go @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package vespa + +import ( + "fmt" + "net" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDetectHostname(t *testing.T) { + lookupAddr := func(addr string) ([]string, error) { + return nil, fmt.Errorf("could not look up %s", addr) + } + lookupIP := func(host string) ([]net.IP, error) { + return nil, fmt.Errorf("no address found for %s", host) + } + + t.Setenv("VESPA_HOSTNAME", "foo.bar") + got, err := findOurHostname(lookupAddr, lookupIP) + assert.Nil(t, err) + assert.Equal(t, "foo.bar", got) + os.Unsetenv("VESPA_HOSTNAME") + got, err = findOurHostnameFrom("bar.foo.123", lookupAddr, lookupIP) + fmt.Fprintln(os.Stderr, "findOurHostname from bar.foo.123 returns:", got, "with error:", err) + assert.NotEqual(t, "", got) + parts := strings.Split(got, ".") + if len(parts) > 1 { + expanded, err2 := findOurHostnameFrom(parts[0], lookupAddr, lookupIP) + fmt.Fprintln(os.Stderr, "findOurHostname from", parts[0], "returns:", expanded, "with error:", err2) + assert.Equal(t, got, expanded) + } + got, err = findOurHostname(lookupAddr, lookupIP) + assert.NotEqual(t, "", got) + fmt.Fprintln(os.Stderr, "findOurHostname() returns:", got, "with error:", err) +} diff --git a/client/go/vespa/load_env.go b/client/go/vespa/load_env.go index a0c127ca920..8eb7c841235 100644 --- a/client/go/vespa/load_env.go +++ b/client/go/vespa/load_env.go @@ -8,11 +8,77 @@ import ( "bufio" "fmt" "os" + "os/user" "strings" ) // backwards-compatible parsing of default-env.txt func LoadDefaultEnv() error { + return loadDefaultEnvTo(new(osEnvReceiver)) +} + +// parse default-env.txt, then dump export statements for "sh" to stdout +func ExportDefaultEnvToSh() error { + holder := newShellEnvExporter() + err := loadDefaultEnvTo(holder) + holder.dump() + return err +} + +// Which user should vespa services run as? If current user is root, +// we want to change to some non-privileged user. +// Should be run after LoadDefaultEnv() which possibly loads VESPA_USER +func FindVespaUser() string { + uName := os.Getenv("VESPA_USER") + if uName != "" { + // no check here, assume valid + return uName + } + if os.Getuid() == 0 { + u, err := user.Lookup("vespa") + if err == nil { + uName = u.Username + } else { + u, err = user.Lookup("nobody") + if err == nil { + uName = u.Username + } + } + } + if uName == "" { + u, err := user.Current() + if err == nil { + uName = u.Username + } + } + if uName != "" { + os.Setenv("VESPA_USER", uName) + } + return uName +} + +type loadEnvReceiver interface { + fallbackVar(varName, varVal string) + overrideVar(varName, varVal string) + unsetVar(varName string) +} + +type osEnvReceiver struct { +} + +func (p *osEnvReceiver) fallbackVar(varName, varVal string) { + if os.Getenv(varName) == "" { + os.Setenv(varName, varVal) + } +} +func (p *osEnvReceiver) overrideVar(varName, varVal string) { + os.Setenv(varName, varVal) +} +func (p *osEnvReceiver) unsetVar(varName string) { + os.Unsetenv(varName) +} + +func loadDefaultEnvTo(r loadEnvReceiver) error { const defEnvTxt = "/conf/vespa/default-env.txt" vespaHome := FindHome() f, err := os.Open(vespaHome + defEnvTxt) @@ -42,13 +108,11 @@ func LoadDefaultEnv() error { } switch action { case "override": - os.Setenv(varName, varVal) + r.overrideVar(varName, varVal) case "fallback": - if os.Getenv(varName) == "" { - os.Setenv(varName, varVal) - } + r.fallbackVar(varName, varVal) case "unset": - os.Unsetenv(varName) + r.unsetVar(varName) default: err = fmt.Errorf("unknown action '%s'", action) } @@ -107,3 +171,93 @@ func isValidShellVariableName(s string) bool { } return len(s) > 0 } + +type shellEnvExporter struct { + exportVars map[string]string + unsetVars map[string]string +} + +func newShellEnvExporter() *shellEnvExporter { + return &shellEnvExporter{ + exportVars: make(map[string]string), + unsetVars: make(map[string]string), + } +} +func (p *shellEnvExporter) fallbackVar(varName, varVal string) { + if os.Getenv(varName) == "" || p.unsetVars[varName] != "" { + delete(p.unsetVars, varName) + p.exportVars[varName] = shellQuote(varVal) + } +} +func (p *shellEnvExporter) overrideVar(varName, varVal string) { + delete(p.unsetVars, varName) + p.exportVars[varName] = shellQuote(varVal) +} +func (p *shellEnvExporter) unsetVar(varName string) { + delete(p.exportVars, varName) + p.unsetVars[varName] = "unset" +} + +func shellQuote(s string) string { + l := 0 + nq := false + for _, ch := range s { + switch { + case (ch >= 'A' && ch <= 'Z') || + (ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9'): + l++ + case ch == '_' || ch == ' ': + l++ + nq = true + case ch == '\'' || ch == '\\': + l = l + 4 + nq = true + default: + l++ + nq = true + } + } + if nq { + l = l + 2 + } + res := make([]rune, l) + i := 0 + if nq { + res[i] = '\'' + i++ + } + for _, ch := range s { + if ch == '\'' || ch == '\\' { + res[i] = '\'' + i++ + res[i] = '\\' + i++ + res[i] = ch + i++ + res[i] = '\'' + } else { + res[i] = ch + } + i++ + } + if nq { + res[i] = '\'' + i++ + } + if i != l { + err := fmt.Errorf("expected length %d but was %d", l, i) + panic(err) + } + return string(res) +} + +func (p *shellEnvExporter) dump() { + for vn, vv := range p.exportVars { + fmt.Printf("%s=%s\n", vn, vv) + fmt.Printf("export %s\n", vn) + } + for vn, _ := range p.unsetVars { + fmt.Printf("unset %s\n", vn) + } +} diff --git a/client/go/vespa/load_env_test.go b/client/go/vespa/load_env_test.go index c5b42cae161..41373f7ab82 100644 --- a/client/go/vespa/load_env_test.go +++ b/client/go/vespa/load_env_test.go @@ -2,6 +2,7 @@ package vespa import ( + "fmt" "os" "testing" @@ -15,15 +16,15 @@ func setup(t *testing.T, contents string) { envf := cdir + "/default-env.txt" err := os.MkdirAll(cdir, 0755) assert.Nil(t, err) - os.Setenv("VESPA_HOME", vdir) + t.Setenv("VESPA_HOME", vdir) err = os.WriteFile(envf, []byte(contents), 0644) assert.Nil(t, err) } func TestLoadEnvSimple(t *testing.T) { - os.Setenv("VESPA_FOO", "was foo") - os.Setenv("VESPA_BAR", "was bar") - os.Setenv("VESPA_FOOBAR", "foobar") + t.Setenv("VESPA_FOO", "was foo") + t.Setenv("VESPA_BAR", "was bar") + t.Setenv("VESPA_FOOBAR", "foobar") os.Unsetenv("VESPA_QUUX") setup(t, ` # vespa env vars file @@ -98,3 +99,59 @@ override VESPA_V2 v2 assert.NotNil(t, err) assert.Equal(t, err.Error(), "Not a valid environment variable name: '.A'") } + +func TestFindUser(t *testing.T) { + u := FindVespaUser() + if u == "" { + fmt.Fprintln(os.Stderr, "WARNING: empty result from FindVespaUser()") + } else { + fmt.Fprintln(os.Stderr, "INFO: result from FindVespaUser() is", u) + assert.Equal(t, u, os.Getenv("VESPA_USER")) + } + setup(t, ` +override VESPA_USER unprivuser +`) + LoadDefaultEnv() + u = FindVespaUser() + assert.Equal(t, "unprivuser", u) +} + +func TestExportEnv(t *testing.T) { + t.Setenv("VESPA_FOO", "was foo") + t.Setenv("VESPA_BAR", "was bar") + t.Setenv("VESPA_FOOBAR", "foobar") + t.Setenv("VESPA_BARFOO", "was barfoo") + os.Unsetenv("VESPA_QUUX") + setup(t, ` +# vespa env vars file +override VESPA_FOO "newFoo1" + +fallback VESPA_BAR "new bar" +fallback VESPA_QUUX "new quux" + +unset VESPA_FOOBAR +unset VESPA_BARFOO +fallback VESPA_BARFOO new'b<a>r'foo +override XYZ xyz +unset XYZ +`) + holder := newShellEnvExporter() + err := loadDefaultEnvTo(holder) + assert.Nil(t, err) + // new values: + assert.Equal(t, "newFoo1", holder.exportVars["VESPA_FOO"]) + assert.Equal(t, "", holder.exportVars["VESPA_BAR"]) + assert.Equal(t, "'new quux'", holder.exportVars["VESPA_QUUX"]) + assert.Equal(t, `'new'\''b<a>r'\''foo'`, holder.exportVars["VESPA_BARFOO"]) + // unsets: + assert.Equal(t, "", holder.exportVars["VESPA_FOOBAR"]) + assert.Equal(t, "unset", holder.unsetVars["VESPA_FOOBAR"]) + assert.Equal(t, "", holder.exportVars["XYZ"]) + assert.Equal(t, "unset", holder.unsetVars["XYZ"]) + // nothing extra allowed: + assert.Equal(t, 3, len(holder.exportVars)) + assert.Equal(t, 2, len(holder.unsetVars)) + // run it + err = ExportDefaultEnvToSh() + assert.Nil(t, err) +} diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 9a2bb770906..34dda889c5a 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -52,6 +52,9 @@ type Target interface { // Type returns this target's type, e.g. local or cloud. Type() string + // IsCloud returns whether this target is Vespa Cloud or hosted Vespa + IsCloud() bool + // Deployment returns the deployment managed by this target. Deployment() Deployment diff --git a/client/go/vespa/target_cloud.go b/client/go/vespa/target_cloud.go index 1076724a252..c89e6f6ecef 100644 --- a/client/go/vespa/target_cloud.go +++ b/client/go/vespa/target_cloud.go @@ -114,6 +114,8 @@ func (t *cloudTarget) Type() string { return TargetCloud } +func (t *cloudTarget) IsCloud() bool { return true } + func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment } func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) { @@ -384,7 +386,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { return err } if len(urlsByCluster) == 0 { - return fmt.Errorf("no endpoints discovered") + return fmt.Errorf("no endpoints discovered for %s", t.deploymentOptions.Deployment) } t.deploymentOptions.ClusterURLs = urlsByCluster return nil diff --git a/client/go/vespa/target_custom.go b/client/go/vespa/target_custom.go index bc25f19bf1a..c34f801641c 100644 --- a/client/go/vespa/target_custom.go +++ b/client/go/vespa/target_custom.go @@ -33,6 +33,8 @@ func CustomTarget(httpClient util.HTTPClient, baseURL string) Target { func (t *customTarget) Type() string { return t.targetType } +func (t *customTarget) IsCloud() bool { return false } + func (t *customTarget) Deployment() Deployment { return Deployment{} } func (t *customTarget) createService(name string) (*Service, error) { diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index f0827aa79d1..4395240acd3 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -51,12 +51,12 @@ public class ClusterController extends AbstractComponent metricWrapper.updateMetricImplementation(metricImpl); verifyThatZooKeeperWorks(options); synchronized (controllers) { - FleetController controller = controllers.get(options.clusterName); + FleetController controller = controllers.get(options.clusterName()); if (controller == null) { StatusHandler.ContainerStatusPageServer statusPageServer = new StatusHandler.ContainerStatusPageServer(); controller = FleetController.create(options, statusPageServer, metricWrapper); - controllers.put(options.clusterName, controller); - status.put(options.clusterName, statusPageServer); + controllers.put(options.clusterName(), controller); + status.put(options.clusterName(), statusPageServer); } else { controller.updateOptions(options); } @@ -120,8 +120,8 @@ public class ClusterController extends AbstractComponent * Block until we are connected to zookeeper server */ private void verifyThatZooKeeperWorks(FleetControllerOptions options) throws Exception { - if (options.zooKeeperServerAddress != null && !"".equals(options.zooKeeperServerAddress)) { - try (Curator curator = Curator.create(options.zooKeeperServerAddress)) { + if (options.zooKeeperServerAddress() != null && !"".equals(options.zooKeeperServerAddress())) { + try (Curator curator = Curator.create(options.zooKeeperServerAddress())) { if ( ! curator.framework().blockUntilConnected(600, TimeUnit.SECONDS)) com.yahoo.protect.Process.logAndDie("Failed to connect to ZK, dying and restarting container"); } diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index bf38cf52aea..016345af8ce 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -55,69 +55,62 @@ public class ClusterControllerClusterConfigurer extends AbstractComponent { SlobroksConfig slobroksConfig, ZookeepersConfig zookeepersConfig) { Distribution distribution = new Distribution(distributionConfig); - FleetControllerOptions options = new FleetControllerOptions(fleetcontrollerConfig.cluster_name(), distribution.getNodes()); - options.setStorageDistribution(distribution); - configure(options, fleetcontrollerConfig); - configure(options, slobroksConfig); - configure(options, zookeepersConfig); - return options; + FleetControllerOptions.Builder builder = new FleetControllerOptions.Builder(fleetcontrollerConfig.cluster_name(), distribution.getNodes()); + builder.setStorageDistribution(distribution); + configure(builder, fleetcontrollerConfig); + configure(builder, slobroksConfig); + configure(builder, zookeepersConfig); + return builder.build(); } - private static void configure(FleetControllerOptions options, FleetcontrollerConfig config) { - options.clusterName = config.cluster_name(); - options.fleetControllerIndex = config.index(); - options.fleetControllerCount = config.fleet_controller_count(); - options.zooKeeperSessionTimeout = (int) (config.zookeeper_session_timeout() * 1000); - options.masterZooKeeperCooldownPeriod = (int) (config.master_zookeeper_cooldown_period() * 1000); - options.stateGatherCount = config.state_gather_count(); - options.rpcPort = config.rpc_port(); - options.httpPort = config.http_port(); - options.maxTransitionTime.put(NodeType.STORAGE, config.storage_transition_time()); - options.maxTransitionTime.put(NodeType.DISTRIBUTOR, config.distributor_transition_time()); - options.maxInitProgressTime = config.init_progress_time(); - options.statePollingFrequency = config.state_polling_frequency(); - options.maxPrematureCrashes = config.max_premature_crashes(); - options.stableStateTimePeriod = config.stable_state_time_period(); - options.eventLogMaxSize = config.event_log_max_size(); - options.eventNodeLogMaxSize = config.event_node_log_max_size(); - options.minDistributorNodesUp = config.min_distributors_up_count(); - options.minStorageNodesUp = config.min_storage_up_count(); - options.minRatioOfDistributorNodesUp = config.min_distributor_up_ratio(); - options.minRatioOfStorageNodesUp = config.min_storage_up_ratio(); - options.cycleWaitTime = (int) (config.cycle_wait_time() * 1000); - options.minTimeBeforeFirstSystemStateBroadcast = (int) (config.min_time_before_first_system_state_broadcast() * 1000); - options.nodeStateRequestTimeoutMS = (int) (config.get_node_state_request_timeout() * 1000); - options.showLocalSystemStatesInEventLog = config.show_local_systemstates_in_event_log(); - options.minTimeBetweenNewSystemStates = config.min_time_between_new_systemstates(); - options.maxSlobrokDisconnectGracePeriod = (int) (config.max_slobrok_disconnect_grace_period() * 1000); - options.distributionBits = config.ideal_distribution_bits(); - options.minNodeRatioPerGroup = config.min_node_ratio_per_group(); - options.setMaxDeferredTaskVersionWaitTime(Duration.ofMillis((int)(config.max_deferred_task_version_wait_time_sec() * 1000))); - options.clusterHasGlobalDocumentTypes = config.cluster_has_global_document_types(); - options.minMergeCompletionRatio = config.min_merge_completion_ratio(); - options.enableTwoPhaseClusterStateActivation = config.enable_two_phase_cluster_state_transitions(); - options.clusterFeedBlockEnabled = config.enable_cluster_feed_block(); - options.clusterFeedBlockLimit = Map.copyOf(config.cluster_feed_block_limit()); - options.clusterFeedBlockNoiseLevel = config.cluster_feed_block_noise_level(); + private static void configure(FleetControllerOptions.Builder builder, FleetcontrollerConfig config) { + builder.setClusterName(config.cluster_name()); + builder.setIndex(config.index()); + builder.setCount(config.fleet_controller_count()); + builder.setZooKeeperSessionTimeout((int) (config.zookeeper_session_timeout() * 1000)); + builder.setMasterZooKeeperCooldownPeriod((int) (config.master_zookeeper_cooldown_period() * 1000)); + builder.setStateGatherCount(config.state_gather_count()); + builder.setRpcPort(config.rpc_port()); + builder.setHttpPort(config.http_port()); + builder.setMaxTransitionTime(NodeType.STORAGE, config.storage_transition_time()); + builder.setMaxTransitionTime(NodeType.DISTRIBUTOR, config.distributor_transition_time()); + builder.setMaxInitProgressTime(config.init_progress_time()); + builder.setStatePollingFrequency(config.state_polling_frequency()); + builder.setMaxPrematureCrashes(config.max_premature_crashes()); + builder.setStableStateTimePeriod(config.stable_state_time_period()); + builder.setEventLogMaxSize(config.event_log_max_size()); + builder.setEventNodeLogMaxSize(config.event_node_log_max_size()); + builder.setMinDistributorNodesUp(config.min_distributors_up_count()); + builder.setMinStorageNodesUp(config.min_storage_up_count()); + builder.setMinRatioOfDistributorNodesUp(config.min_distributor_up_ratio()); + builder.setMinRatioOfStorageNodesUp(config.min_storage_up_ratio()); + builder.setCycleWaitTime((int) (config.cycle_wait_time() * 1000)); + builder.setMinTimeBeforeFirstSystemStateBroadcast((int) (config.min_time_before_first_system_state_broadcast() * 1000)); + builder.setNodeStateRequestTimeoutMS((int) (config.get_node_state_request_timeout() * 1000)); + builder.setShowLocalSystemStatesInEventLog(config.show_local_systemstates_in_event_log()); + builder.setMinTimeBetweenNewSystemStates(config.min_time_between_new_systemstates()); + builder.setMaxSlobrokDisconnectGracePeriod((int) (config.max_slobrok_disconnect_grace_period() * 1000)); + builder.setDistributionBits(config.ideal_distribution_bits()); + builder.setMinNodeRatioPerGroup(config.min_node_ratio_per_group()); + builder.setMaxDeferredTaskVersionWaitTime(Duration.ofMillis((int)(config.max_deferred_task_version_wait_time_sec() * 1000))); + builder.setClusterHasGlobalDocumentTypes(config.cluster_has_global_document_types()); + builder.setMinMergeCompletionRatio(config.min_merge_completion_ratio()); + builder.enableTwoPhaseClusterStateActivation(config.enable_two_phase_cluster_state_transitions()); + builder.setClusterFeedBlockEnabled(config.enable_cluster_feed_block()); + builder.setClusterFeedBlockLimit(Map.copyOf(config.cluster_feed_block_limit())); + builder.setClusterFeedBlockNoiseLevel(config.cluster_feed_block_noise_level()); } - private static void configure(FleetControllerOptions options, SlobroksConfig config) { + private static void configure(FleetControllerOptions.Builder builder, SlobroksConfig config) { String[] specs = new String[config.slobrok().size()]; for (int i = 0; i < config.slobrok().size(); i++) { specs[i] = config.slobrok().get(i).connectionspec(); } - options.slobrokConnectionSpecs = specs; + builder.setSlobrokConnectionSpecs(specs); } - private static void configure(FleetControllerOptions options, ZookeepersConfig config) { - options.zooKeeperServerAddress = verifyZooKeeperAddress(config.zookeeperserverlist()); - } - - private static String verifyZooKeeperAddress(String zooKeeperServerAddress) { - if (zooKeeperServerAddress == null || "".equals(zooKeeperServerAddress)) { - throw new IllegalArgumentException("zookeeper server address must be set, was '" + zooKeeperServerAddress + "'"); - } - return zooKeeperServerAddress; + private static void configure(FleetControllerOptions.Builder builder, ZookeepersConfig config) { + builder.setZooKeeperServerAddress(config.zookeeperserverlist()); } } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java index 7c5913774c3..11caf0397e0 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java @@ -66,11 +66,11 @@ public class ClusterControllerClusterConfigurerTest { null ); assertNotNull(configurer.getOptions()); - assertEquals(0.123, configurer.getOptions().minNodeRatioPerGroup, 0.01); - assertTrue(configurer.getOptions().clusterFeedBlockEnabled); - assertEquals(0.5, configurer.getOptions().clusterFeedBlockLimit.get("foo"), 0.01); - assertEquals(0.7, configurer.getOptions().clusterFeedBlockLimit.get("bar"), 0.01); - assertEquals(0.05, configurer.getOptions().clusterFeedBlockNoiseLevel, 0.001); + assertEquals(0.123, configurer.getOptions().minNodeRatioPerGroup(), 0.01); + assertTrue(configurer.getOptions().clusterFeedBlockEnabled()); + assertEquals(0.5, configurer.getOptions().clusterFeedBlockLimit().get("foo"), 0.01); + assertEquals(0.7, configurer.getOptions().clusterFeedBlockLimit().get("bar"), 0.01); + assertEquals(0.05, configurer.getOptions().clusterFeedBlockNoiseLevel(), 0.001); try { zookeepersConfig.zookeeperserverlist(""); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java index 403c5c6089b..e62190b0ec8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java @@ -112,14 +112,14 @@ public class ClusterStateGenerator { */ static Params fromOptions(FleetControllerOptions opts) { return new Params() - .maxPrematureCrashes(opts.maxPrematureCrashes) - .minStorageNodesUp(opts.minStorageNodesUp) - .minDistributorNodesUp(opts.minDistributorNodesUp) - .minRatioOfStorageNodesUp(opts.minRatioOfStorageNodesUp) - .minRatioOfDistributorNodesUp(opts.minRatioOfDistributorNodesUp) - .minNodeRatioPerGroup(opts.minNodeRatioPerGroup) - .idealDistributionBits(opts.distributionBits) - .transitionTimes(opts.maxTransitionTime); + .maxPrematureCrashes(opts.maxPrematureCrashes()) + .minStorageNodesUp(opts.minStorageNodesUp()) + .minDistributorNodesUp(opts.minDistributorNodesUp()) + .minRatioOfStorageNodesUp(opts.minRatioOfStorageNodesUp()) + .minRatioOfDistributorNodesUp(opts.minRatioOfDistributorNodesUp()) + .minNodeRatioPerGroup(opts.minNodeRatioPerGroup()) + .idealDistributionBits(opts.distributionBits()) + .transitionTimes(opts.maxTransitionTime()); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 3f3bf62bf4d..f58dfe553bf 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -37,6 +37,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.TimeZone; @@ -128,14 +129,11 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta this.stateGatherer = nodeStateGatherer; this.stateChangeHandler = stateChangeHandler; this.systemStateBroadcaster = systemStateBroadcaster; - this.stateVersionTracker = new StateVersionTracker(options.minMergeCompletionRatio); + this.stateVersionTracker = new StateVersionTracker(options.minMergeCompletionRatio()); this.metricUpdater = metricUpdater; - - this.statusPageServer = statusPage; + this.statusPageServer = Objects.requireNonNull(statusPage, "statusPage cannot be null"); this.rpcServer = server; - this.masterElectionHandler = masterElectionHandler; - this.statusRequestRouter.addHandler( "^/node=([a-z]+)\\.(\\d+)$", new LegacyNodePageRequestHandler(timer, eventLog, cluster)); @@ -157,26 +155,23 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta MetricReporter metricReporter) throws Exception { var context = new FleetControllerContextImpl(options); var timer = new RealTimer(); - var metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex, options.clusterName); + var metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex(), options.clusterName()); var log = new EventLog(timer, metricUpdater); - var cluster = new ContentCluster( - options.clusterName, - options.nodes, - options.storageDistribution); + var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); var stateGatherer = new NodeStateGatherer(timer, timer, log); var communicator = new RPCCommunicator( RPCCommunicator.createRealSupervisor(), timer, - options.fleetControllerIndex, - options.nodeStateRequestTimeoutMS, - options.nodeStateRequestTimeoutEarliestPercentage, - options.nodeStateRequestTimeoutLatestPercentage, - options.nodeStateRequestRoundTripTimeMaxSeconds); - var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress, timer); + options.fleetControllerIndex(), + options.nodeStateRequestTimeoutMS(), + options.nodeStateRequestTimeoutEarliestPercentage(), + options.nodeStateRequestTimeoutLatestPercentage(), + options.nodeStateRequestRoundTripTimeMaxSeconds()); + var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress(), timer); var lookUp = new SlobrokClient(context, timer); var stateGenerator = new StateChangeHandler(context, timer, log); var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); - var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex, options.fleetControllerCount, timer, timer); + var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex(), options.fleetControllerCount(), timer, timer); var controller = new FleetController(context, timer, log, cluster, stateGatherer, communicator, statusPageServer, null, lookUp, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); @@ -237,7 +232,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta public FleetControllerOptions getOptions() { synchronized(monitor) { - return options.clone(); + return FleetControllerOptions.Builder.copy(options).build(); } } @@ -277,9 +272,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta controllerThreadId = Thread.currentThread().getId(); database.shutdown(databaseContext); - if (statusPageServer != null) { - statusPageServer.shutdown(); - } + statusPageServer.shutdown(); if (rpcServer != null) { rpcServer.shutdown(); } @@ -292,7 +285,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta synchronized(monitor) { assert newId.equals(context.id()); context.log(logger, Level.INFO, "FleetController has new options"); - nextOptions = options.clone(); + nextOptions = FleetControllerOptions.Builder.copy(options).build(); monitor.notifyAll(); } } @@ -335,7 +328,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } private void triggerBundleRecomputationIfResourceExhaustionStateChanged(NodeInfo nodeInfo, HostInfo newHostInfo) { - if (!options.clusterFeedBlockEnabled) { + if (!options.clusterFeedBlockEnabled()) { return; } var calc = createResourceExhaustionCalculator(); @@ -376,7 +369,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta ClusterState baselineState = stateBundle.getBaselineClusterState(); newStates.add(stateBundle); metricUpdater.updateClusterStateMetrics(cluster, baselineState, - ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit, stateBundle.getFeedBlock())); + ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit(), stateBundle.getFeedBlock())); lastMetricUpdateCycleCount = cycleCount; systemStateBroadcaster.handleNewClusterStates(stateBundle); // Iff master, always store new version in ZooKeeper _before_ publishing to any @@ -395,7 +388,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta ClusterStateBundle stateBundle = stateVersionTracker.getVersionedClusterStateBundle(); ClusterState baselineState = stateBundle.getBaselineClusterState(); metricUpdater.updateClusterStateMetrics(cluster, baselineState, - ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit, stateBundle.getFeedBlock())); + ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit(), stateBundle.getFeedBlock())); lastMetricUpdateCycleCount = cycleCount; return true; } else { @@ -489,61 +482,59 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta verifyInControllerThread(); selfTerminateIfConfiguredNodeIndexHasChanged(); - if (changesConfiguredNodeSet(options.nodes)) { + if (changesConfiguredNodeSet(options.nodes())) { // Force slobrok node re-fetch in case of changes to the set of configured nodes cluster.setSlobrokGenerationCount(0); } configuredBucketSpaces = Set.of(FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace()); - stateVersionTracker.setMinMergeCompletionRatio(options.minMergeCompletionRatio); + stateVersionTracker.setMinMergeCompletionRatio(options.minMergeCompletionRatio()); communicator.propagateOptions(options); if (nodeLookup instanceof SlobrokClient) { - ((SlobrokClient) nodeLookup).setSlobrokConnectionSpecs(options.slobrokConnectionSpecs); + ((SlobrokClient) nodeLookup).setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()); } - eventLog.setMaxSize(options.eventLogMaxSize, options.eventNodeLogMaxSize); - cluster.setPollingFrequency(options.statePollingFrequency); - cluster.setDistribution(options.storageDistribution); - cluster.setNodes(options.nodes, databaseContext.getNodeStateUpdateListener()); - database.setZooKeeperAddress(options.zooKeeperServerAddress, databaseContext); - database.setZooKeeperSessionTimeout(options.zooKeeperSessionTimeout, databaseContext); - stateGatherer.setMaxSlobrokDisconnectGracePeriod(options.maxSlobrokDisconnectGracePeriod); - stateGatherer.setNodeStateRequestTimeout(options.nodeStateRequestTimeoutMS); + eventLog.setMaxSize(options.eventLogMaxSize(), options.eventNodeLogMaxSize()); + cluster.setPollingFrequency(options.statePollingFrequency()); + cluster.setDistribution(options.storageDistribution()); + cluster.setNodes(options.nodes(), databaseContext.getNodeStateUpdateListener()); + database.setZooKeeperAddress(options.zooKeeperServerAddress(), databaseContext); + database.setZooKeeperSessionTimeout(options.zooKeeperSessionTimeout(), databaseContext); + stateGatherer.setMaxSlobrokDisconnectGracePeriod(options.maxSlobrokDisconnectGracePeriod()); + stateGatherer.setNodeStateRequestTimeout(options.nodeStateRequestTimeoutMS()); // TODO: remove as many temporal parameter dependencies as possible here. Currently duplication of state. stateChangeHandler.reconfigureFromOptions(options); stateChangeHandler.setStateChangedFlag(); // Always trigger state recomputation after reconfig - masterElectionHandler.setFleetControllerCount(options.fleetControllerCount); - masterElectionHandler.setMasterZooKeeperCooldownPeriod(options.masterZooKeeperCooldownPeriod); - masterElectionHandler.setUsingZooKeeper(options.zooKeeperServerAddress != null && !options.zooKeeperServerAddress.isEmpty()); + masterElectionHandler.setFleetControllerCount(options.fleetControllerCount()); + masterElectionHandler.setMasterZooKeeperCooldownPeriod(options.masterZooKeeperCooldownPeriod()); + masterElectionHandler.setUsingZooKeeper(options.zooKeeperServerAddress() != null && !options.zooKeeperServerAddress().isEmpty()); if (rpcServer != null) { rpcServer.setMasterElectionHandler(masterElectionHandler); try{ - rpcServer.setSlobrokConnectionSpecs(options.slobrokConnectionSpecs, options.rpcPort); + rpcServer.setSlobrokConnectionSpecs(options.slobrokConnectionSpecs(), options.rpcPort()); } catch (ListenFailedException e) { - context.log(logger, Level.WARNING, "Failed to bind RPC server to port " + options.rpcPort + ". This may be natural if cluster has altered the services running on this node: " + e.getMessage()); + context.log(logger, Level.WARNING, "Failed to bind RPC server to port " + options.rpcPort() + ". This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } catch (Exception e) { context.log(logger, Level.WARNING, "Failed to initialize RPC server socket: " + e.getMessage()); } } - if (statusPageServer != null) { - try{ - statusPageServer.setPort(options.httpPort); - } catch (Exception e) { - context.log(logger, Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); - } + try { + statusPageServer.setPort(options.httpPort()); + } catch (Exception e) { + context.log(logger, Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } long currentTime = timer.getCurrentTimeInMillis(); - nextStateSendTime = Math.min(currentTime + options.minTimeBetweenNewSystemStates, nextStateSendTime); + nextStateSendTime = Math.min(currentTime + options.minTimeBetweenNewSystemStates(), nextStateSendTime); } private void selfTerminateIfConfiguredNodeIndexHasChanged() { - var newId = new FleetControllerId(options.clusterName, options.fleetControllerIndex); + var newId = new FleetControllerId(options.clusterName(), options.fleetControllerIndex()); if (!newId.equals(context.id())) { context.log(logger, Level.WARNING, context.id() + " got new configuration for " + newId + ". We do not support doing this live; " + "immediately exiting now to force new configuration"); @@ -603,7 +594,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta if ( ! isRunning()) { return; } - if (masterElectionHandler.isAmongNthFirst(options.stateGatherCount)) { + if (masterElectionHandler.isAmongNthFirst(options.stateGatherCount())) { didWork |= resyncLocallyCachedState(); // Calls to metricUpdate.forWork inside method } else { stepDownAsStateGatherer(); @@ -636,7 +627,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta metricUpdater.addTickTime(tickStopTime - tickStartTime, didWork); } // Always sleep some to use avoid using too much CPU and avoid starving waiting threads - monitor.wait(didWork || waitingForCycle ? 1 : options.cycleWaitTime); + monitor.wait(didWork || waitingForCycle ? 1 : options.cycleWaitTime()); if ( ! isRunning()) { return; } tickStartTime = timer.getCurrentTimeInMillis(); processingCycle = true; @@ -679,12 +670,10 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } private boolean processAnyPendingStatusPageRequest() { - if (statusPageServer != null) { - StatusPageServer.HttpRequest statusRequest = statusPageServer.getCurrentHttpRequest(); - if (statusRequest != null) { - statusPageServer.answerCurrentStatusRequest(fetchStatusPage(statusRequest)); - return true; - } + StatusPageServer.HttpRequest statusRequest = statusPageServer.getCurrentHttpRequest(); + if (statusRequest != null) { + statusPageServer.answerCurrentStatusRequest(fetchStatusPage(statusRequest)); + return true; } return false; } @@ -715,7 +704,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta databaseContext, communicator, database.getLastKnownStateBundleVersionWrittenBySelf()); if (sentAny) { // FIXME won't this inhibit resending to unresponsive nodes? - nextStateSendTime = currentTime + options.minTimeBetweenNewSystemStates; + nextStateSendTime = currentTime + options.minTimeBetweenNewSystemStates(); } } // Always allow activations if we've already broadcasted a state @@ -823,7 +812,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta return ""; } return String.format("the following nodes have not converged to at least version %d: %s", - taskConvergeVersion, stringifyListWithLimits(nodes, options.maxDivergentNodesPrintedInTaskErrorMessages)); + taskConvergeVersion, stringifyListWithLimits(nodes, options.maxDivergentNodesPrintedInTaskErrorMessages())); } private boolean completeSatisfiedVersionDependentTasks() { @@ -931,7 +920,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta final ClusterStateBundle candidateBundle = ClusterStateBundle.builder(candidate) .bucketSpaces(configuredBucketSpaces) .stateDeriver(createBucketSpaceStateDeriver()) - .deferredActivation(options.enableTwoPhaseClusterStateActivation) + .deferredActivation(options.enableTwoPhaseClusterStateActivation()) .feedBlock(createResourceExhaustionCalculator() .inferContentClusterFeedBlockOrNull(cluster.getNodeInfos())) .deriveAndBuild(); @@ -964,7 +953,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } private ClusterStateDeriver createBucketSpaceStateDeriver() { - if (options.clusterHasGlobalDocumentTypes) { + if (options.clusterHasGlobalDocumentTypes()) { return new MaintenanceWhenPendingGlobalMerges(stateVersionTracker.createMergePendingChecker(), createDefaultSpaceMaintenanceTransitionConstraint()); } else { @@ -973,10 +962,10 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } private ResourceExhaustionCalculator createResourceExhaustionCalculator() { - return new ResourceExhaustionCalculator( - options.clusterFeedBlockEnabled, options.clusterFeedBlockLimit, - stateVersionTracker.getLatestCandidateStateBundle().getFeedBlockOrNull(), - options.clusterFeedBlockNoiseLevel); + return new ResourceExhaustionCalculator(options.clusterFeedBlockEnabled(), + options.clusterFeedBlockLimit(), + stateVersionTracker.getLatestCandidateStateBundle().getFeedBlockOrNull(), + options.clusterFeedBlockNoiseLevel()); } private static ClusterStateDeriver createIdentityClonedBucketSpaceStateDeriver() { @@ -1081,11 +1070,11 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta eventLog.add(new ClusterEvent(ClusterEvent.Type.MASTER_ELECTION, "This node just became fleetcontroller master. Bumped version to " + stateVersionTracker.getCurrentVersion() + " to be in line.", timer.getCurrentTimeInMillis())); long currentTime = timer.getCurrentTimeInMillis(); - firstAllowedStateBroadcast = currentTime + options.minTimeBeforeFirstSystemStateBroadcast; + firstAllowedStateBroadcast = currentTime + options.minTimeBeforeFirstSystemStateBroadcast(); isMaster = true; inMasterMoratorium = true; context.log(logger, Level.FINE, () -> "At time " + currentTime + " we set first system state broadcast time to be " - + options.minTimeBeforeFirstSystemStateBroadcast + " ms after at time " + firstAllowedStateBroadcast + "."); + + options.minTimeBeforeFirstSystemStateBroadcast() + " ms after at time " + firstAllowedStateBroadcast + "."); didWork = true; } if (wantedStateChanged) { @@ -1147,19 +1136,24 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta public NodeListener getNodeStateUpdateListener() { return FleetController.this; } }; + // For testing only public void waitForCompleteCycle(Duration timeout) { Instant endTime = Instant.now().plus(timeout); synchronized (monitor) { // To wait at least one complete cycle, if a cycle is already running we need to wait for the next one beyond. long wantedCycle = cycleCount + (processingCycle ? 2 : 1); waitingForCycle = true; - try{ + try { while (cycleCount < wantedCycle) { if (Instant.now().isAfter(endTime)) throw new IllegalStateException("Timed out waiting for cycle to complete. Not completed after " + timeout); if ( !isRunning() ) throw new IllegalStateException("Fleetcontroller not running. Will never complete cycles"); - try{ monitor.wait(100); } catch (InterruptedException e) {} + try { + monitor.wait(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } finally { waitingForCycle = false; @@ -1224,8 +1218,6 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta public ContentCluster getCluster() { return cluster; } - public List<NodeEvent> getNodeEvents(Node n) { return eventLog.getNodeEvents(n); } - public EventLog getEventLog() { return eventLog; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerId.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerId.java index fd0784d69c9..5a5d245eb69 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerId.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerId.java @@ -13,7 +13,7 @@ public class FleetControllerId { private final int index; public static FleetControllerId fromOptions(FleetControllerOptions options) { - return new FleetControllerId(options.clusterName, options.fleetControllerIndex); + return new FleetControllerId(options.clusterName(), options.fleetControllerIndex()); } public FleetControllerId(String clusterName, int index) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 16e0e1c0673..6666066422a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -5,75 +5,68 @@ import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.NodeType; - -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; import java.time.Duration; import java.util.Collection; import java.util.Collections; -import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.stream.Collectors; /** - * This class represents all the options that can be set in the fleetcontroller. + * Immutable class representing all the options that can be set in the fleetcontroller. * Tests typically just generate an instance of this object to use in fleet controller for testing. - * A real application generate this object from config, and on config updates, post new options to the fleet controller. + * A real application generates this object from config, and on config updates, post new options to the fleet controller. */ -public class FleetControllerOptions implements Cloneable { - - // TODO: Make fields private +public class FleetControllerOptions { - public String fleetControllerConfigId; - public String slobrokConfigId; + private final String fleetControllerConfigId; + private final String slobrokConfigId; - public String clusterName; - public int fleetControllerIndex = 0; - public int fleetControllerCount = 1; - public int stateGatherCount = 2; + private final String clusterName; + private final int fleetControllerIndex; + private final int fleetControllerCount; + private final int stateGatherCount; - // TODO: This cannot be null but nonnull is not verified - public String[] slobrokConnectionSpecs; - public int rpcPort = 0; - public int httpPort = 0; - public int distributionBits = 16; + private final String[] slobrokConnectionSpecs; + private final int rpcPort; + private final int httpPort; + private final int distributionBits; /** Timeout before breaking zookeeper session (in milliseconds) */ - public int zooKeeperSessionTimeout = 5 * 60 * 1000; + private final int zooKeeperSessionTimeout; /** * Timeout between master disappearing before new master will take over. * (Grace period to allow old master to detect that it is disconnected from zookeeper) */ - public int masterZooKeeperCooldownPeriod = 15 * 1000; + private final int masterZooKeeperCooldownPeriod; - public String zooKeeperServerAddress = null; + private final String zooKeeperServerAddress; - public int statePollingFrequency = 5000; + private final int statePollingFrequency; /** * Max amount of time to keep a node, that has previously been available * in steady state, in maintenance mode, while node is unreachable, before setting it down. */ - public Map<NodeType, Integer> maxTransitionTime = new TreeMap<>(); + private final Map<NodeType, Integer> maxTransitionTime; /** * Max amount of time to keep a storage node, that is initializing, in maintenance mode, without any further * initializing progress being received, before setting it down. */ - public int maxInitProgressTime = 5000; + private final int maxInitProgressTime; - public int maxPrematureCrashes = 4; - public long stableStateTimePeriod = 2 * 60 * 60 * 1000; + private final int maxPrematureCrashes; + private final long stableStateTimePeriod; - public int eventLogMaxSize = 1024; - public int eventNodeLogMaxSize = 1024; + private final int eventLogMaxSize; + private final int eventNodeLogMaxSize; - public int minDistributorNodesUp = 1; - public int minStorageNodesUp = 1; - public double minRatioOfDistributorNodesUp = 0.50; - public double minRatioOfStorageNodesUp = 0.50; + private final int minDistributorNodesUp; + private final int minStorageNodesUp; + private final double minRatioOfDistributorNodesUp; + private final double minRatioOfStorageNodesUp; /** * Minimum ratio of nodes in an "available" state (up, initializing or maintenance) @@ -83,13 +76,13 @@ public class FleetControllerOptions implements Cloneable { * * A value of 0.0 implies group auto-takedown feature is effectively disabled. */ - public double minNodeRatioPerGroup = 0.0; + private final double minNodeRatioPerGroup; /** * Milliseconds to sleep after doing a work cycle where we did no work. Some events do not interrupt the sleeping, * such as slobrok changes, so shouldn't set this too high. */ - public int cycleWaitTime = 100; + private final int cycleWaitTime; /** * Minimum time to pass (in milliseconds) before broadcasting our first systemstate. Set small in unit tests, * but should be a few seconds in a real system to prevent new nodes taking over from disturbing the system by @@ -98,7 +91,7 @@ public class FleetControllerOptions implements Cloneable { * reported their state in Slobrok and getnodestate. This value should typically be in the order of * maxSlobrokDisconnectGracePeriod and nodeStateRequestTimeoutMS. */ - public long minTimeBeforeFirstSystemStateBroadcast = 0; + private final long minTimeBeforeFirstSystemStateBroadcast; /** * StateRequestTimeout for the request are randomized a bit to avoid congestion on replies. The effective @@ -106,151 +99,729 @@ public class FleetControllerOptions implements Cloneable { * [nodeStateRequestTimeoutEarliestPercentage * nodeStateRequestTimeoutMS / 100, * nodeStateRequestTimeoutLatestPercentage * nodeStateRequestTimeoutMS / 100]. */ - public int nodeStateRequestTimeoutMS = 5 * 60 * 1000; - public int nodeStateRequestTimeoutEarliestPercentage = 80; - public int nodeStateRequestTimeoutLatestPercentage = 95; - public int nodeStateRequestRoundTripTimeMaxSeconds = 5; + private final int nodeStateRequestTimeoutMS; + private final int nodeStateRequestTimeoutEarliestPercentage; + private final int nodeStateRequestTimeoutLatestPercentage; + private final int nodeStateRequestRoundTripTimeMaxSeconds; - public int minTimeBetweenNewSystemStates = 0; - public boolean showLocalSystemStatesInEventLog = true; + private final int minTimeBetweenNewSystemStates; + private final boolean showLocalSystemStatesInEventLog; /** Maximum time a node can be missing from slobrok before it is tagged down. */ - public int maxSlobrokDisconnectGracePeriod = 1000; + private final int maxSlobrokDisconnectGracePeriod; /** Set by tests to retry often. */ - public BackOffPolicy slobrokBackOffPolicy = null; + private final BackOffPolicy slobrokBackOffPolicy; - public Distribution storageDistribution; + private final Distribution storageDistribution; // TODO: Get rid of this by always getting nodes by distribution.getNodes() - public Set<ConfiguredNode> nodes; + private final Set<ConfiguredNode> nodes; - private Duration maxDeferredTaskVersionWaitTime = Duration.ofSeconds(30); + private final Duration maxDeferredTaskVersionWaitTime; - public boolean clusterHasGlobalDocumentTypes = false; + private final boolean clusterHasGlobalDocumentTypes; - public boolean enableTwoPhaseClusterStateActivation = false; + private final boolean enableTwoPhaseClusterStateActivation; - // TODO: Choose a default value - public double minMergeCompletionRatio = 1.0; + private final double minMergeCompletionRatio; - public int maxDivergentNodesPrintedInTaskErrorMessages = 10; + private final int maxDivergentNodesPrintedInTaskErrorMessages; - public boolean clusterFeedBlockEnabled = false; + private final boolean clusterFeedBlockEnabled; // Resource type -> limit in [0, 1] - public Map<String, Double> clusterFeedBlockLimit = Collections.emptyMap(); + private final Map<String, Double> clusterFeedBlockLimit; + + private final double clusterFeedBlockNoiseLevel; + + private FleetControllerOptions(String fleetControllerConfigId, + String slobrokConfigId, + String clusterName, + int fleetControllerIndex, + int fleetControllerCount, + int stateGatherCount, + String[] slobrokConnectionSpecs, + int rpcPort, + int httpPort, + int distributionBits, + int zooKeeperSessionTimeout, + int masterZooKeeperCooldownPeriod, + String zooKeeperServerAddress, + int statePollingFrequency, + Map<NodeType, Integer> maxTransitionTime, + int maxInitProgressTime, + int maxPrematureCrashes, + long stableStateTimePeriod, + int eventLogMaxSize, + int eventNodeLogMaxSize, + int minDistributorNodesUp, + int minStorageNodesUp, + double minRatioOfDistributorNodesUp, + double minRatioOfStorageNodesUp, + double minNodeRatioPerGroup, + int cycleWaitTime, + long minTimeBeforeFirstSystemStateBroadcast, + int nodeStateRequestTimeoutMS, + int nodeStateRequestTimeoutEarliestPercentage, + int nodeStateRequestTimeoutLatestPercentage, + int nodeStateRequestRoundTripTimeMaxSeconds, + int minTimeBetweenNewSystemStates, + boolean showLocalSystemStatesInEventLog, + int maxSlobrokDisconnectGracePeriod, + BackOffPolicy slobrokBackOffPolicy, + Distribution storageDistribution, + Set<ConfiguredNode> nodes, + Duration maxDeferredTaskVersionWaitTime, + boolean clusterHasGlobalDocumentTypes, + boolean enableTwoPhaseClusterStateActivation, + double minMergeCompletionRatio, + int maxDivergentNodesPrintedInTaskErrorMessages, + boolean clusterFeedBlockEnabled, + Map<String, Double> clusterFeedBlockLimit, + double clusterFeedBlockNoiseLevel) { + this.fleetControllerConfigId = fleetControllerConfigId; + this.slobrokConfigId = slobrokConfigId; + this.clusterName = clusterName; + this.fleetControllerIndex = fleetControllerIndex; + this.fleetControllerCount = fleetControllerCount; + this.stateGatherCount = stateGatherCount; + this.slobrokConnectionSpecs = slobrokConnectionSpecs; + this.rpcPort = rpcPort; + this.httpPort = httpPort; + this.distributionBits = distributionBits; + this.zooKeeperSessionTimeout = zooKeeperSessionTimeout; + this.masterZooKeeperCooldownPeriod = masterZooKeeperCooldownPeriod; + this.zooKeeperServerAddress = zooKeeperServerAddress; + this.statePollingFrequency = statePollingFrequency; + this.maxTransitionTime = maxTransitionTime; + this.maxInitProgressTime = maxInitProgressTime; + this.maxPrematureCrashes = maxPrematureCrashes; + this.stableStateTimePeriod = stableStateTimePeriod; + this.eventLogMaxSize = eventLogMaxSize; + this.eventNodeLogMaxSize = eventNodeLogMaxSize; + this.minDistributorNodesUp = minDistributorNodesUp; + this.minStorageNodesUp = minStorageNodesUp; + this.minRatioOfDistributorNodesUp = minRatioOfDistributorNodesUp; + this.minRatioOfStorageNodesUp = minRatioOfStorageNodesUp; + this.minNodeRatioPerGroup = minNodeRatioPerGroup; + this.cycleWaitTime = cycleWaitTime; + this.minTimeBeforeFirstSystemStateBroadcast = minTimeBeforeFirstSystemStateBroadcast; + this.nodeStateRequestTimeoutMS = nodeStateRequestTimeoutMS; + this.nodeStateRequestTimeoutEarliestPercentage = nodeStateRequestTimeoutEarliestPercentage; + this.nodeStateRequestTimeoutLatestPercentage = nodeStateRequestTimeoutLatestPercentage; + this.nodeStateRequestRoundTripTimeMaxSeconds = nodeStateRequestRoundTripTimeMaxSeconds; + this.minTimeBetweenNewSystemStates = minTimeBetweenNewSystemStates; + this.showLocalSystemStatesInEventLog = showLocalSystemStatesInEventLog; + this.maxSlobrokDisconnectGracePeriod = maxSlobrokDisconnectGracePeriod; + this.slobrokBackOffPolicy = slobrokBackOffPolicy; + this.storageDistribution = storageDistribution; + this.nodes = nodes; + this.maxDeferredTaskVersionWaitTime = maxDeferredTaskVersionWaitTime; + this.clusterHasGlobalDocumentTypes = clusterHasGlobalDocumentTypes; + this.enableTwoPhaseClusterStateActivation = enableTwoPhaseClusterStateActivation; + this.minMergeCompletionRatio = minMergeCompletionRatio; + this.maxDivergentNodesPrintedInTaskErrorMessages = maxDivergentNodesPrintedInTaskErrorMessages; + this.clusterFeedBlockEnabled = clusterFeedBlockEnabled; + this.clusterFeedBlockLimit = clusterFeedBlockLimit; + this.clusterFeedBlockNoiseLevel = clusterFeedBlockNoiseLevel; + } - public double clusterFeedBlockNoiseLevel = 0.01; + public Duration getMaxDeferredTaskVersionWaitTime() { + return maxDeferredTaskVersionWaitTime; + } - public FleetControllerOptions(String clusterName, Collection<ConfiguredNode> nodes) { - this.clusterName = clusterName; - maxTransitionTime.put(NodeType.DISTRIBUTOR, 0); - maxTransitionTime.put(NodeType.STORAGE, 5000); - this.nodes = new TreeSet<>(nodes); + public long storageNodeMaxTransitionTimeMs() { + return maxTransitionTime.getOrDefault(NodeType.STORAGE, 10_000); } - /** Called on reconfiguration of this cluster */ - public void setStorageDistribution(Distribution distribution) { - this.storageDistribution = distribution; + public String fleetControllerConfigId() {return fleetControllerConfigId;} + + public String slobrokConfigId() { + return slobrokConfigId; } - public Duration getMaxDeferredTaskVersionWaitTime() { + public String clusterName() { + return clusterName; + } + + public int fleetControllerIndex() { + return fleetControllerIndex; + } + + public int fleetControllerCount() { + return fleetControllerCount; + } + + public int stateGatherCount() { + return stateGatherCount; + } + + public String[] slobrokConnectionSpecs() { + return slobrokConnectionSpecs; + } + + public int rpcPort() { + return rpcPort; + } + + public int httpPort() { + return httpPort; + } + + public int distributionBits() { + return distributionBits; + } + + public int zooKeeperSessionTimeout() { + return zooKeeperSessionTimeout; + } + + public int masterZooKeeperCooldownPeriod() { + return masterZooKeeperCooldownPeriod; + } + + public String zooKeeperServerAddress() { + return zooKeeperServerAddress; + } + + public int statePollingFrequency() { + return statePollingFrequency; + } + + public Map<NodeType, Integer> maxTransitionTime() { + return maxTransitionTime; + } + + public int maxInitProgressTime() { + return maxInitProgressTime; + } + + public int maxPrematureCrashes() { + return maxPrematureCrashes; + } + + public long stableStateTimePeriod() { + return stableStateTimePeriod; + } + + public int eventLogMaxSize() { + return eventLogMaxSize; + } + + public int eventNodeLogMaxSize() { + return eventNodeLogMaxSize; + } + + public int minDistributorNodesUp() { + return minDistributorNodesUp; + } + + public int minStorageNodesUp() { + return minStorageNodesUp; + } + + public double minRatioOfDistributorNodesUp() { + return minRatioOfDistributorNodesUp; + } + + public double minRatioOfStorageNodesUp() { + return minRatioOfStorageNodesUp; + } + + public double minNodeRatioPerGroup() { + return minNodeRatioPerGroup; + } + + public int cycleWaitTime() { + return cycleWaitTime; + } + + public long minTimeBeforeFirstSystemStateBroadcast() { + return minTimeBeforeFirstSystemStateBroadcast; + } + + public int nodeStateRequestTimeoutMS() { + return nodeStateRequestTimeoutMS; + } + + public int nodeStateRequestTimeoutEarliestPercentage() { + return nodeStateRequestTimeoutEarliestPercentage; + } + + public int nodeStateRequestTimeoutLatestPercentage() { + return nodeStateRequestTimeoutLatestPercentage; + } + + public int nodeStateRequestRoundTripTimeMaxSeconds() { + return nodeStateRequestRoundTripTimeMaxSeconds; + } + + public int minTimeBetweenNewSystemStates() { + return minTimeBetweenNewSystemStates; + } + + public boolean showLocalSystemStatesInEventLog() { + return showLocalSystemStatesInEventLog; + } + + public int maxSlobrokDisconnectGracePeriod() { + return maxSlobrokDisconnectGracePeriod; + } + + public BackOffPolicy slobrokBackOffPolicy() { + return slobrokBackOffPolicy; + } + + public Distribution storageDistribution() { + return storageDistribution; + } + + public Set<ConfiguredNode> nodes() { + return nodes; + } + + public Duration maxDeferredTaskVersionWaitTime() { return maxDeferredTaskVersionWaitTime; } - public void setMaxDeferredTaskVersionWaitTime(Duration maxDeferredTaskVersionWaitTime) { - this.maxDeferredTaskVersionWaitTime = maxDeferredTaskVersionWaitTime; + public boolean clusterHasGlobalDocumentTypes() { + return clusterHasGlobalDocumentTypes; } - public long storageNodeMaxTransitionTimeMs() { - return maxTransitionTime.getOrDefault(NodeType.STORAGE, 10_000); + public boolean enableTwoPhaseClusterStateActivation() { + return enableTwoPhaseClusterStateActivation; } - public FleetControllerOptions clone() { - try { - // TODO: This should deep clone - return (FleetControllerOptions) super.clone(); - } catch (CloneNotSupportedException e) { - throw new RuntimeException("Will not happen"); - } + public double minMergeCompletionRatio() { + return minMergeCompletionRatio; + } + + public int maxDivergentNodesPrintedInTaskErrorMessages() { + return maxDivergentNodesPrintedInTaskErrorMessages; + } + + public boolean clusterFeedBlockEnabled() { + return clusterFeedBlockEnabled; } - public static String splitZooKeeperAddress(String s) { - StringBuilder sb = new StringBuilder(); - while (true) { - int index = s.indexOf(','); - if (index > 0) { - sb.append(s.substring(0, index + 1)).append(' '); - s = s.substring(index+1); - } else { - break; + public Map<String, Double> clusterFeedBlockLimit() { + return clusterFeedBlockLimit; + } + + public double clusterFeedBlockNoiseLevel() { + return clusterFeedBlockNoiseLevel; + } + + public static class Builder { + + private String fleetControllerConfigId; + private String slobrokConfigId; + private String clusterName; + private int index = 0; + private int count = 1; + private int stateGatherCount = 2; + private String[] slobrokConnectionSpecs; + private int rpcPort = 0; + private int httpPort = 0; + private int distributionBits = 16; + private int zooKeeperSessionTimeout = 5 * 60 * 1000; + private int masterZooKeeperCooldownPeriod = 15 * 1000; + private String zooKeeperServerAddress = null; + private int statePollingFrequency = 5000; + private Map<NodeType, Integer> maxTransitionTime = new TreeMap<>(); + private int maxInitProgressTime = 5000; + private int maxPrematureCrashes = 4; + private long stableStateTimePeriod = 2 * 60 * 60 * 1000; + private int eventLogMaxSize = 1024; + private int eventNodeLogMaxSize = 1024; + private int minDistributorNodesUp = 1; + private int minStorageNodesUp = 1; + private double minRatioOfDistributorNodesUp = 0.50; + private double minRatioOfStorageNodesUp = 0.50; + private double minNodeRatioPerGroup = 0.0; + private int cycleWaitTime = 100; + private long minTimeBeforeFirstSystemStateBroadcast = 0; + private int nodeStateRequestTimeoutMS = 5 * 60 * 1000; + private int nodeStateRequestTimeoutEarliestPercentage = 80; + private int nodeStateRequestTimeoutLatestPercentage = 95; + private int nodeStateRequestRoundTripTimeMaxSeconds = 5; + private int minTimeBetweenNewSystemStates = 0; + private boolean showLocalSystemStatesInEventLog = true; + private int maxSlobrokDisconnectGracePeriod = 1000; + private BackOffPolicy slobrokBackOffPolicy = null; + private Distribution storageDistribution; + private Set<ConfiguredNode> nodes; + private Duration maxDeferredTaskVersionWaitTime = Duration.ofSeconds(30); + private boolean clusterHasGlobalDocumentTypes = false; + private boolean enableTwoPhaseClusterStateActivation = false; + private double minMergeCompletionRatio = 1.0; + private int maxDivergentNodesPrintedInTaskErrorMessages = 10; + private boolean clusterFeedBlockEnabled = false; + private Map<String, Double> clusterFeedBlockLimit = Collections.emptyMap(); + private double clusterFeedBlockNoiseLevel = 0.01; + + public Builder(String clusterName, Collection<ConfiguredNode> nodes) { + this.clusterName = clusterName; + this.nodes = new TreeSet<>(nodes); + maxTransitionTime.put(NodeType.DISTRIBUTOR, 0); + maxTransitionTime.put(NodeType.STORAGE, 5000); + } + + public String clusterName() { + return clusterName; + } + + public Builder setClusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } + + public int fleetControllerIndex() { + return index; + } + + public Builder setIndex(int index) { + this.index = index; + return this; + } + + public Builder setCount(int count) { + this.count = count; + return this; + } + + public Builder setStateGatherCount(int stateGatherCount) { + this.stateGatherCount = stateGatherCount; + return this; + } + + public String[] slobrokConnectionSpecs() { + return slobrokConnectionSpecs; + } + + public Builder setSlobrokConnectionSpecs(String[] slobrokConnectionSpecs) { + Objects.requireNonNull(slobrokConnectionSpecs, "slobrokConnectionSpecs cannot be null"); + this.slobrokConnectionSpecs = slobrokConnectionSpecs; + return this; + } + + public Builder setRpcPort(int rpcPort) { + this.rpcPort = rpcPort; + return this; + } + + public Builder setHttpPort(int httpPort) { + this.httpPort = httpPort; + return this; + } + + public Builder setDistributionBits(int distributionBits) { + this.distributionBits = distributionBits; + return this; + } + + public Builder setZooKeeperSessionTimeout(int zooKeeperSessionTimeout) { + this.zooKeeperSessionTimeout = zooKeeperSessionTimeout; + return this; + } + + public Builder setMasterZooKeeperCooldownPeriod(int masterZooKeeperCooldownPeriod) { + this.masterZooKeeperCooldownPeriod = masterZooKeeperCooldownPeriod; + return this; + } + + public String zooKeeperServerAddress() { + return zooKeeperServerAddress; + } + + public Builder setZooKeeperServerAddress(String zooKeeperServerAddress) { + if (zooKeeperServerAddress == null || "".equals(zooKeeperServerAddress)) { + throw new IllegalArgumentException("zookeeper server address must be set, was '" + zooKeeperServerAddress + "'"); } + this.zooKeeperServerAddress = zooKeeperServerAddress; + return this; + } + + public Builder setStatePollingFrequency(int statePollingFrequency) { + this.statePollingFrequency = statePollingFrequency; + return this; + } + + public Map<NodeType, Integer> maxTransitionTime() { + return maxTransitionTime; + } + + public Builder setMaxTransitionTime(NodeType nodeType, Integer maxTransitionTime) { + this.maxTransitionTime.put(nodeType, maxTransitionTime); + return this; + } + + public int maxInitProgressTime() { + return maxInitProgressTime; + } + + public Builder setMaxInitProgressTime(int maxInitProgressTime) { + this.maxInitProgressTime = maxInitProgressTime; + return this; + } + + public int maxPrematureCrashes() { + return maxPrematureCrashes; + } + + public Builder setMaxPrematureCrashes(int maxPrematureCrashes) { + this.maxPrematureCrashes = maxPrematureCrashes; + return this; + } + + public long stableStateTimePeriod() { + return stableStateTimePeriod; + } + + public Builder setStableStateTimePeriod(long stableStateTimePeriod) { + this.stableStateTimePeriod = stableStateTimePeriod; + return this; + } + + public Builder setEventLogMaxSize(int eventLogMaxSize) { + this.eventLogMaxSize = eventLogMaxSize; + return this; + } + + public Builder setEventNodeLogMaxSize(int eventNodeLogMaxSize) { + this.eventNodeLogMaxSize = eventNodeLogMaxSize; + return this; + } + + public Builder setMinDistributorNodesUp(int minDistributorNodesUp) { + this.minDistributorNodesUp = minDistributorNodesUp; + return this; + } + + public Builder setMinStorageNodesUp(int minStorageNodesUp) { + this.minStorageNodesUp = minStorageNodesUp; + return this; + } + + public Builder setMinRatioOfDistributorNodesUp(double minRatioOfDistributorNodesUp) { + this.minRatioOfDistributorNodesUp = minRatioOfDistributorNodesUp; + return this; + } + + public Builder setMinRatioOfStorageNodesUp(double minRatioOfStorageNodesUp) { + this.minRatioOfStorageNodesUp = minRatioOfStorageNodesUp; + return this; + } + + public Builder setMinNodeRatioPerGroup(double minNodeRatioPerGroup) { + this.minNodeRatioPerGroup = minNodeRatioPerGroup; + return this; + } + + public Builder setCycleWaitTime(int cycleWaitTime) { + this.cycleWaitTime = cycleWaitTime; + return this; + } + + public Builder setMinTimeBeforeFirstSystemStateBroadcast(long minTimeBeforeFirstSystemStateBroadcast) { + this.minTimeBeforeFirstSystemStateBroadcast = minTimeBeforeFirstSystemStateBroadcast; + return this; + } + + public int nodeStateRequestTimeoutMS() { + return nodeStateRequestTimeoutMS; + } + + public Builder setNodeStateRequestTimeoutMS(int nodeStateRequestTimeoutMS) { + this.nodeStateRequestTimeoutMS = nodeStateRequestTimeoutMS; + return this; + } + + public Builder setNodeStateRequestTimeoutEarliestPercentage(int nodeStateRequestTimeoutEarliestPercentage) { + this.nodeStateRequestTimeoutEarliestPercentage = nodeStateRequestTimeoutEarliestPercentage; + return this; + } + + public Builder setNodeStateRequestTimeoutLatestPercentage(int nodeStateRequestTimeoutLatestPercentage) { + this.nodeStateRequestTimeoutLatestPercentage = nodeStateRequestTimeoutLatestPercentage; + return this; + } + + public Builder setMinTimeBetweenNewSystemStates(int minTimeBetweenNewSystemStates) { + this.minTimeBetweenNewSystemStates = minTimeBetweenNewSystemStates; + return this; + } + + public Builder setShowLocalSystemStatesInEventLog(boolean showLocalSystemStatesInEventLog) { + this.showLocalSystemStatesInEventLog = showLocalSystemStatesInEventLog; + return this; + } + + public int maxSlobrokDisconnectGracePeriod() { + return maxSlobrokDisconnectGracePeriod; + } + + public Builder setMaxSlobrokDisconnectGracePeriod(int maxSlobrokDisconnectGracePeriod) { + this.maxSlobrokDisconnectGracePeriod = maxSlobrokDisconnectGracePeriod; + return this; + } + + public Builder setStorageDistribution(Distribution storageDistribution) { + this.storageDistribution = storageDistribution; + return this; + } + + public Set<ConfiguredNode> nodes() { + return nodes; + } + + public Builder setNodes(Set<ConfiguredNode> nodes) { + this.nodes = nodes; + return this; + } + + public Builder setMaxDeferredTaskVersionWaitTime(Duration maxDeferredTaskVersionWaitTime) { + this.maxDeferredTaskVersionWaitTime = maxDeferredTaskVersionWaitTime; + return this; + } + + public Builder setClusterHasGlobalDocumentTypes(boolean clusterHasGlobalDocumentTypes) { + this.clusterHasGlobalDocumentTypes = clusterHasGlobalDocumentTypes; + return this; + } + + public Builder enableTwoPhaseClusterStateActivation(boolean enableTwoPhaseClusterStateActivation) { + this.enableTwoPhaseClusterStateActivation = enableTwoPhaseClusterStateActivation; + return this; + } + + public double minMergeCompletionRatio() { + return minMergeCompletionRatio; + } + + public Builder setMinMergeCompletionRatio(double minMergeCompletionRatio) { + this.minMergeCompletionRatio = minMergeCompletionRatio; + return this; + } + + public Builder setMaxDivergentNodesPrintedInTaskErrorMessages(int maxDivergentNodesPrintedInTaskErrorMessages) { + this.maxDivergentNodesPrintedInTaskErrorMessages = maxDivergentNodesPrintedInTaskErrorMessages; + return this; + } + + public Builder setClusterFeedBlockEnabled(boolean clusterFeedBlockEnabled) { + this.clusterFeedBlockEnabled = clusterFeedBlockEnabled; + return this; + } + + public Builder setClusterFeedBlockLimit(Map<String, Double> clusterFeedBlockLimit) { + this.clusterFeedBlockLimit = Map.copyOf(clusterFeedBlockLimit); + return this; + } + + public Builder setClusterFeedBlockNoiseLevel(double clusterFeedBlockNoiseLevel) { + this.clusterFeedBlockNoiseLevel = clusterFeedBlockNoiseLevel; + return this; + } + + public FleetControllerOptions build() { + return new FleetControllerOptions(fleetControllerConfigId, + slobrokConfigId, + clusterName, + index, + count, + stateGatherCount, + slobrokConnectionSpecs, + rpcPort, + httpPort, + distributionBits, + zooKeeperSessionTimeout, + masterZooKeeperCooldownPeriod, + zooKeeperServerAddress, + statePollingFrequency, + maxTransitionTime, + maxInitProgressTime, + maxPrematureCrashes, + stableStateTimePeriod, + eventLogMaxSize, + eventNodeLogMaxSize, + minDistributorNodesUp, + minStorageNodesUp, + minRatioOfDistributorNodesUp, + minRatioOfStorageNodesUp, + minNodeRatioPerGroup, + cycleWaitTime, + minTimeBeforeFirstSystemStateBroadcast, + nodeStateRequestTimeoutMS, + nodeStateRequestTimeoutEarliestPercentage, + nodeStateRequestTimeoutLatestPercentage, + nodeStateRequestRoundTripTimeMaxSeconds, + minTimeBetweenNewSystemStates, + showLocalSystemStatesInEventLog, + maxSlobrokDisconnectGracePeriod, + slobrokBackOffPolicy, + storageDistribution, + nodes, + maxDeferredTaskVersionWaitTime, + clusterHasGlobalDocumentTypes, + enableTwoPhaseClusterStateActivation, + minMergeCompletionRatio, + maxDivergentNodesPrintedInTaskErrorMessages, + clusterFeedBlockEnabled, + clusterFeedBlockLimit, + clusterFeedBlockNoiseLevel); + } + + public static Builder copy(FleetControllerOptions options) { + Builder builder = new Builder(options.clusterName(), options.nodes()); + builder.fleetControllerConfigId = options.fleetControllerConfigId; + builder.slobrokConfigId = options.slobrokConfigId; + builder.clusterName = options.clusterName; + builder.index = options.fleetControllerIndex; + builder.count = options.fleetControllerCount; + builder.stateGatherCount = options.stateGatherCount; + builder.slobrokConnectionSpecs = options.slobrokConnectionSpecs; + builder.rpcPort = options.rpcPort; + builder.httpPort = options.httpPort; + builder.distributionBits = options.distributionBits; + builder.zooKeeperSessionTimeout = options.zooKeeperSessionTimeout; + builder.masterZooKeeperCooldownPeriod = options.masterZooKeeperCooldownPeriod; + builder.zooKeeperServerAddress = options.zooKeeperServerAddress; + builder.statePollingFrequency = options.statePollingFrequency; + builder.maxTransitionTime = Map.copyOf(options.maxTransitionTime); + builder.maxInitProgressTime = options.maxInitProgressTime; + builder.maxPrematureCrashes = options.maxPrematureCrashes; + builder.stableStateTimePeriod = options.stableStateTimePeriod; + builder.eventLogMaxSize = options.eventLogMaxSize; + builder.eventNodeLogMaxSize = options.eventNodeLogMaxSize; + builder.minDistributorNodesUp = options.minDistributorNodesUp; + builder.minStorageNodesUp = options.minStorageNodesUp; + builder.minRatioOfDistributorNodesUp = options.minRatioOfStorageNodesUp; + builder.minRatioOfStorageNodesUp = options.minRatioOfStorageNodesUp; + builder.minNodeRatioPerGroup = options.minNodeRatioPerGroup; + builder.cycleWaitTime = options.cycleWaitTime; + builder.minTimeBeforeFirstSystemStateBroadcast = options.minTimeBeforeFirstSystemStateBroadcast; + builder.nodeStateRequestTimeoutMS = options.nodeStateRequestTimeoutMS; + builder.nodeStateRequestTimeoutEarliestPercentage = options.nodeStateRequestTimeoutEarliestPercentage; + builder.nodeStateRequestTimeoutLatestPercentage = options.nodeStateRequestTimeoutLatestPercentage; + builder.nodeStateRequestRoundTripTimeMaxSeconds = options.nodeStateRequestRoundTripTimeMaxSeconds; + builder.minTimeBetweenNewSystemStates = options.minTimeBetweenNewSystemStates; + builder.showLocalSystemStatesInEventLog = options.showLocalSystemStatesInEventLog; + builder.maxSlobrokDisconnectGracePeriod = options.maxSlobrokDisconnectGracePeriod; + builder.slobrokBackOffPolicy = options.slobrokBackOffPolicy; + builder.storageDistribution = options.storageDistribution; + builder.nodes = Set.copyOf(options.nodes); + builder.maxDeferredTaskVersionWaitTime = options.maxDeferredTaskVersionWaitTime; + builder.clusterHasGlobalDocumentTypes = options.clusterHasGlobalDocumentTypes; + builder.enableTwoPhaseClusterStateActivation = options.enableTwoPhaseClusterStateActivation; + builder.minMergeCompletionRatio = options.minMergeCompletionRatio; + builder.maxDivergentNodesPrintedInTaskErrorMessages = options.maxDivergentNodesPrintedInTaskErrorMessages; + builder.clusterFeedBlockEnabled = options.clusterFeedBlockEnabled; + builder.clusterFeedBlockLimit = Map.copyOf(options.clusterFeedBlockLimit); + builder.clusterFeedBlockNoiseLevel = options.clusterFeedBlockNoiseLevel; + + return builder; } - sb.append(s); - return sb.toString(); - } - - static DecimalFormat DecimalDot2 = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); - - public void writeHtmlState(StringBuilder sb) { - String slobrokspecs = ""; - for (int i=0; i<slobrokConnectionSpecs.length; ++i) { - if (i != 0) slobrokspecs += "<br>"; - slobrokspecs += slobrokConnectionSpecs[i]; - } - sb.append("<h1>Current config</h1>\n") - .append("<p>Fleet controller config id: ").append(fleetControllerConfigId == null ? null : fleetControllerConfigId.replaceAll("\n", "<br>\n")).append("</p>\n") - .append("<p>Slobrok config id: ").append(slobrokConfigId == null ? null : slobrokConfigId.replaceAll("\n", "<br>\n")).append("</p>\n") - .append("<table border=\"1\" cellspacing=\"0\"><tr><th>Property</th><th>Value</th></tr>\n"); - - sb.append("<tr><td><nobr>Cluster name</nobr></td><td align=\"right\">").append(clusterName).append("</td></tr>"); - sb.append("<tr><td><nobr>Fleet controller index</nobr></td><td align=\"right\">").append(fleetControllerIndex).append("/").append(fleetControllerCount).append("</td></tr>"); - sb.append("<tr><td><nobr>Number of fleetcontrollers gathering states from nodes</nobr></td><td align=\"right\">").append(stateGatherCount).append("</td></tr>"); - - sb.append("<tr><td><nobr>Slobrok connection spec</nobr></td><td align=\"right\">").append(slobrokspecs).append("</td></tr>"); - sb.append("<tr><td><nobr>RPC port</nobr></td><td align=\"right\">").append(rpcPort == 0 ? "Pick random available" : rpcPort).append("</td></tr>"); - sb.append("<tr><td><nobr>HTTP port</nobr></td><td align=\"right\">").append(httpPort == 0 ? "Pick random available" : httpPort).append("</td></tr>"); - sb.append("<tr><td><nobr>Master cooldown period</nobr></td><td align=\"right\">").append(RealTimer.printDuration(masterZooKeeperCooldownPeriod)).append("</td></tr>"); - String zooKeeperAddress = (zooKeeperServerAddress == null ? "Not using Zookeeper" : splitZooKeeperAddress(zooKeeperServerAddress)); - sb.append("<tr><td><nobr>Zookeeper server address</nobr></td><td align=\"right\">").append(zooKeeperAddress).append("</td></tr>"); - sb.append("<tr><td><nobr>Zookeeper session timeout</nobr></td><td align=\"right\">").append(RealTimer.printDuration(zooKeeperSessionTimeout)).append("</td></tr>"); - - sb.append("<tr><td><nobr>Cycle wait time</nobr></td><td align=\"right\">").append(cycleWaitTime).append(" ms</td></tr>"); - sb.append("<tr><td><nobr>Minimum time before first clusterstate broadcast as master</nobr></td><td align=\"right\">").append(RealTimer.printDuration(minTimeBeforeFirstSystemStateBroadcast)).append("</td></tr>"); - sb.append("<tr><td><nobr>Minimum time between official cluster states</nobr></td><td align=\"right\">").append(RealTimer.printDuration(minTimeBetweenNewSystemStates)).append("</td></tr>"); - sb.append("<tr><td><nobr>Slobrok mirror backoff policy</nobr></td><td align=\"right\">").append(slobrokBackOffPolicy == null ? "default" : "overridden").append("</td></tr>"); - - sb.append("<tr><td><nobr>Node state request timeout</nobr></td><td align=\"right\">").append(RealTimer.printDuration(nodeStateRequestTimeoutMS)).append("</td></tr>"); - sb.append("<tr><td><nobr>VDS 4.1 node state polling frequency</nobr></td><td align=\"right\">").append(RealTimer.printDuration(statePollingFrequency)).append("</td></tr>"); - sb.append("<tr><td><nobr>Maximum distributor transition time</nobr></td><td align=\"right\">").append(RealTimer.printDuration(maxTransitionTime.get(NodeType.DISTRIBUTOR))).append("</td></tr>"); - sb.append("<tr><td><nobr>Maximum storage transition time</nobr></td><td align=\"right\">").append(RealTimer.printDuration(maxTransitionTime.get(NodeType.STORAGE))).append("</td></tr>"); - sb.append("<tr><td><nobr>Maximum initialize without progress time</nobr></td><td align=\"right\">").append(RealTimer.printDuration(maxInitProgressTime)).append("</td></tr>"); - sb.append("<tr><td><nobr>Maximum premature crashes</nobr></td><td align=\"right\">").append(maxPrematureCrashes).append("</td></tr>"); - sb.append("<tr><td><nobr>Stable state time period</nobr></td><td align=\"right\">").append(RealTimer.printDuration(stableStateTimePeriod)).append("</td></tr>"); - sb.append("<tr><td><nobr>Slobrok disconnect grace period</nobr></td><td align=\"right\">").append(RealTimer.printDuration(maxSlobrokDisconnectGracePeriod)).append("</td></tr>"); - - sb.append("<tr><td><nobr>Number of distributor nodes</nobr></td><td align=\"right\">").append(nodes == null ? "Autodetect" : nodes.size()).append("</td></tr>"); - sb.append("<tr><td><nobr>Number of storage nodes</nobr></td><td align=\"right\">").append(nodes == null ? "Autodetect" : nodes.size()).append("</td></tr>"); - sb.append("<tr><td><nobr>Minimum distributor nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(minDistributorNodesUp).append("</td></tr>"); - sb.append("<tr><td><nobr>Minimum storage nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(minStorageNodesUp).append("</td></tr>"); - sb.append("<tr><td><nobr>Minimum percentage of distributor nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(DecimalDot2.format(100 * minRatioOfDistributorNodesUp)).append(" %</td></tr>"); - sb.append("<tr><td><nobr>Minimum percentage of storage nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(DecimalDot2.format(100 * minRatioOfStorageNodesUp)).append(" %</td></tr>"); - - sb.append("<tr><td><nobr>Show local cluster state changes</nobr></td><td align=\"right\">").append(showLocalSystemStatesInEventLog).append("</td></tr>"); - sb.append("<tr><td><nobr>Maximum event log size</nobr></td><td align=\"right\">").append(eventLogMaxSize).append("</td></tr>"); - sb.append("<tr><td><nobr>Maximum node event log size</nobr></td><td align=\"right\">").append(eventNodeLogMaxSize).append("</td></tr>"); - sb.append("<tr><td><nobr>Wanted distribution bits</nobr></td><td align=\"right\">").append(distributionBits).append("</td></tr>"); - sb.append("<tr><td><nobr>Max deferred task version wait time</nobr></td><td align=\"right\">").append(maxDeferredTaskVersionWaitTime.toMillis()).append("ms</td></tr>"); - sb.append("<tr><td><nobr>Cluster has global document types configured</nobr></td><td align=\"right\">").append(clusterHasGlobalDocumentTypes).append("</td></tr>"); - sb.append("<tr><td><nobr>Enable 2-phase cluster state activation protocol</nobr></td><td align=\"right\">").append(enableTwoPhaseClusterStateActivation).append("</td></tr>"); - sb.append("<tr><td><nobr>Cluster auto feed block on resource exhaustion enabled</nobr></td><td align=\"right\">") - .append(clusterFeedBlockEnabled).append("</td></tr>"); - sb.append("<tr><td><nobr>Feed block limits</nobr></td><td align=\"right\">") - .append(clusterFeedBlockLimit.entrySet().stream() - .map(kv -> String.format("%s: %.2f%%", kv.getKey(), kv.getValue() * 100.0)) - .collect(Collectors.joining("<br/>"))).append("</td></tr>"); - - sb.append("</table>"); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 637aca16ee7..dfc328346bb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -35,11 +35,9 @@ public class MasterElectionHandler implements MasterInterface { this.index = index; this.totalCount = totalCount; this.nextInLineCount = Integer.MAX_VALUE; - // Only a given set of nodes can ever become master - if (index > (totalCount - 1) / 2) { + if (cannotBecomeMaster()) context.log(logger, Level.FINE, () -> "We can never become master and will always stay a follower."); - } - // Tag current time as when we have not seen any other master. Make sure we're not taking over at once for master that is on the way down + // Tag current time as when we have not seen any other master. Make sure we're not taking over at once for master that is on the way down masterGoneFromZooKeeperTime = timer.getCurrentTimeInMillis(); } @@ -77,16 +75,15 @@ public class MasterElectionHandler implements MasterInterface { @Override public Integer getMaster() { - // If too few followers there can be no master - if (2 * followers <= totalCount) { + if (tooFewFollowersToHaveAMaster()) { return null; } - // If all are following master candidate, it is master if it exists. + // If all are following master candidate, it is master if it exists. if (followers == totalCount) { return masterCandidate; } - // If not all are following we only accept master candidate if old master - // disappeared sufficient time ago + // If not all are following we only accept master candidate if old master + // disappeared sufficient time ago if (masterGoneFromZooKeeperTime + masterZooKeeperCooldownPeriod > timer.getCurrentTimeInMillis()) { return null; } @@ -97,8 +94,7 @@ public class MasterElectionHandler implements MasterInterface { if (masterCandidate == null) { return "There is currently no master candidate."; } - // If too few followers there can be no master - if (2 * followers <= totalCount) { + if (tooFewFollowersToHaveAMaster()) { return "More than half of the nodes must agree for there to be a master. Only " + followers + " of " + totalCount + " nodes agree on current master candidate (" + masterCandidate + ")."; } @@ -118,6 +114,10 @@ public class MasterElectionHandler implements MasterInterface { return followers + " of " + totalCount + " nodes agree " + masterCandidate + " is master."; } + private boolean tooFewFollowersToHaveAMaster() { + return 2 * followers <= totalCount; + } + public boolean isAmongNthFirst(int first) { return (nextInLineCount < first); } public boolean watchMasterElection(DatabaseHandler database, @@ -131,8 +131,8 @@ public class MasterElectionHandler implements MasterInterface { } return false; // Nothing have happened since last time. } - // Move next data to temporary, such that we don't need to keep lock, and such that we don't retry - // if we happen to fail processing the data. + // Move next data to temporary, such that we don't need to keep lock, and such that we don't retry + // if we happen to fail processing the data. Map<Integer, Integer> state; context.log(logger, Level.INFO, "Handling new master election, as we have received " + nextMasterData.size() + " entries"); synchronized (monitor) { @@ -184,8 +184,7 @@ public class MasterElectionHandler implements MasterInterface { database.setMasterVote(dbContext, first.getKey()); } } - // Only a given set of nodes can ever become master - if (index <= (totalCount - 1) / 2) { + if (canBecomeMaster()) { int ourPosition = 0; for (Map.Entry<Integer, Integer> entry : state.entrySet()) { if (entry.getKey() != index) { @@ -205,6 +204,11 @@ public class MasterElectionHandler implements MasterInterface { return true; } + // Only a given set of nodes can ever become master + private boolean canBecomeMaster() {return index <= (totalCount - 1) / 2;} + + private boolean cannotBecomeMaster() {return ! canBecomeMaster();} + private static String toString(Map<Integer, Integer> data) { StringBuilder sb = new StringBuilder(); for (Map.Entry<Integer, Integer> entry : data.entrySet()) { @@ -253,10 +257,10 @@ public class MasterElectionHandler implements MasterInterface { Integer master = getMaster(); if (master != null) { sb.append("<p>Current cluster controller master is node " + master + "."); - if (master.intValue() == index) sb.append(" (This node)"); + if (master == index) sb.append(" (This node)"); sb.append("</p>"); } else { - if (2 * followers <= totalCount) { + if (tooFewFollowersToHaveAMaster()) { sb.append("<p>There is currently no master. Less than half the fleet controllers (") .append(followers).append(") are following master candidate ").append(masterCandidate) .append(".</p>"); @@ -267,19 +271,19 @@ public class MasterElectionHandler implements MasterInterface { .append(" before electing new master unless all possible master candidates are online.</p>"); } } - if ((master == null || master.intValue() != index) && nextInLineCount < stateGatherCount) { + if ((master == null || master != index) && nextInLineCount < stateGatherCount) { sb.append("<p>As we are number ").append(nextInLineCount) .append(" in line for taking over as master, we're gathering state from nodes.</p>"); sb.append("<p><font color=\"red\">As we are not the master, we don't know about nodes current system state" + " or wanted states, so some statistics below may be stale. Look at status page on master " + "for updated data.</font></p>"); } - if (index * 2 > totalCount) { + if (cannotBecomeMaster()) { sb.append("<p>As lowest index fleet controller is prioritized to become master, and more than half " + "of the fleet controllers need to be available to select a master, we can never become master.</p>"); } - // Debug data + // Debug data sb.append("<p><font size=\"-1\" color=\"grey\">Master election handler internal state:") .append("<br>Index: " + index) .append("<br>Fleet controller count: " + totalCount) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 9897e3cf04c..4ab80ec6d7a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -228,11 +228,11 @@ public class StateChangeHandler { } void reconfigureFromOptions(FleetControllerOptions options) { - setMaxPrematureCrashes(options.maxPrematureCrashes); - setStableStateTimePeriod(options.stableStateTimePeriod); - setMaxInitProgressTime(options.maxInitProgressTime); - setMaxSlobrokDisconnectGracePeriod(options.maxSlobrokDisconnectGracePeriod); - setMaxTransitionTime(options.maxTransitionTime); + setMaxPrematureCrashes(options.maxPrematureCrashes()); + setStableStateTimePeriod(options.stableStateTimePeriod()); + setMaxInitProgressTime(options.maxInitProgressTime()); + setMaxSlobrokDisconnectGracePeriod(options.maxSlobrokDisconnectGracePeriod()); + setMaxTransitionTime(options.maxTransitionTime()); } // TODO too many hidden behavior dependencies between this and the actually diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index e223ad12fb9..3738a17f4b6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -30,8 +30,8 @@ import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkArgument; /** - * Responsible for doing RPC requests to VDS nodes. - This class is not thread-safe. + * Responsible for doing RPC requests to storage nodes. + * This class is not thread-safe. */ public class RPCCommunicator implements Communicator { @@ -56,7 +56,6 @@ public class RPCCommunicator implements Communicator { public static Supervisor createRealSupervisor() { return new Supervisor(new Transport("rpc-communicator")).setDropEmptyBuffers(true); - } public RPCCommunicator(Supervisor supervisor, @@ -95,17 +94,16 @@ public class RPCCommunicator implements Communicator { @Override public void propagateOptions(FleetControllerOptions options) { - checkArgument(options.nodeStateRequestTimeoutMS > 0); - checkArgument(options.nodeStateRequestTimeoutEarliestPercentage >= 0); - checkArgument(options.nodeStateRequestTimeoutEarliestPercentage <= 100); - checkArgument(options.nodeStateRequestTimeoutLatestPercentage - >= options.nodeStateRequestTimeoutEarliestPercentage); - checkArgument(options.nodeStateRequestTimeoutLatestPercentage <= 100); - checkArgument(options.nodeStateRequestRoundTripTimeMaxSeconds >= 0); - this.nodeStateRequestTimeoutIntervalMax = Duration.ofMillis(options.nodeStateRequestTimeoutMS); - this.nodeStateRequestTimeoutIntervalStartPercentage = options.nodeStateRequestTimeoutEarliestPercentage; - this.nodeStateRequestTimeoutIntervalStopPercentage = options.nodeStateRequestTimeoutLatestPercentage; - this.nodeStateRequestRoundTripTimeMax = Duration.ofSeconds(options.nodeStateRequestRoundTripTimeMaxSeconds); + checkArgument(options.nodeStateRequestTimeoutMS() > 0); + checkArgument(options.nodeStateRequestTimeoutEarliestPercentage() >= 0); + checkArgument(options.nodeStateRequestTimeoutEarliestPercentage() <= 100); + checkArgument(options.nodeStateRequestTimeoutLatestPercentage() >= options.nodeStateRequestTimeoutEarliestPercentage()); + checkArgument(options.nodeStateRequestTimeoutLatestPercentage() <= 100); + checkArgument(options.nodeStateRequestRoundTripTimeMaxSeconds() >= 0); + this.nodeStateRequestTimeoutIntervalMax = Duration.ofMillis(options.nodeStateRequestTimeoutMS()); + this.nodeStateRequestTimeoutIntervalStartPercentage = options.nodeStateRequestTimeoutEarliestPercentage(); + this.nodeStateRequestTimeoutIntervalStopPercentage = options.nodeStateRequestTimeoutLatestPercentage(); + this.nodeStateRequestRoundTripTimeMax = Duration.ofSeconds(options.nodeStateRequestRoundTripTimeMaxSeconds()); } @Override diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index 4f20b3d0cdc..a7ec2ddaa8f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -20,16 +20,22 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TimeZone; import java.util.TreeMap; +import java.util.stream.Collectors; /** * @author Haakon Humberset */ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHandler { + private static final DecimalFormat DecimalDot2 = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); + private final Timer timer; private final ContentCluster cluster; private final MasterElectionHandler masterElectionHandler; @@ -62,28 +68,27 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa response.setContentType("text/html"); StringBuilder content = new StringBuilder(); content.append("<!-- Answer to request " + request + " -->\n"); - response.writeHtmlHeader(content, cluster.getName() + " Cluster Controller " + options.fleetControllerIndex + " Status Page"); + response.writeHtmlHeader(content, cluster.getName() + " Cluster Controller " + options.fleetControllerIndex() + " Status Page"); content.append("<p><font size=\"-1\">") .append(" [ <a href=\"#config\">Current config</a>") .append(" | <a href=\"#clusterstates\">Cluster states</a>") .append(" | <a href=\"#eventlog\">Event log</a>") .append(" ]</font></p>\n"); content.append("<table><tr><td>UTC time when creating this page:</td><td align=\"right\">").append(RealTimer.printDateNoMilliSeconds(currentTime, tz)).append("</td></tr>"); - //content.append("<tr><td>Fleetcontroller version:</td><td align=\"right\">" + Vtag.V_TAG_PKG + "</td></tr/>"); content.append("<tr><td>Cluster controller uptime:</td><td align=\"right\">" + RealTimer.printDuration(currentTime - startedTime) + "</td></tr></table>"); - if (masterElectionHandler.isAmongNthFirst(options.stateGatherCount)) { + if (masterElectionHandler.isAmongNthFirst(options.stateGatherCount())) { // Table overview of all the nodes writeHtmlState(cluster, content, timer, stateVersionTracker, options, eventLog); // Current cluster state and cluster state history writeHtmlState(stateVersionTracker, content); } else { // Overview of current config - options.writeHtmlState(content); + writeHtmlState(content, options); } // State of master election - masterElectionHandler.writeHtmlState(content, options.stateGatherCount); + masterElectionHandler.writeHtmlState(content, options.stateGatherCount()); // Overview of current config - options.writeHtmlState(content); + writeHtmlState(content, options); // Event log eventLog.writeHtmlState(content, null); response.writeHtmlFooter(content, ""); @@ -167,7 +172,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa HtmlTable.escape(state.getFeedBlockOrNull().getDescription()))); } - List<Group> groups = LeafGroups.enumerateFrom(options.storageDistribution.getRootGroup()); + List<Group> groups = LeafGroups.enumerateFrom(options.storageDistribution().getRootGroup()); for (Group group : groups) { assert (group != null); @@ -184,14 +189,14 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa timer, state, stateVersionTracker.getAggregatedClusterStats(), - options.minMergeCompletionRatio, - options.maxPrematureCrashes, - options.clusterFeedBlockLimit, + options.minMergeCompletionRatio(), + options.maxPrematureCrashes(), + options.clusterFeedBlockLimit(), eventLog, cluster.getName(), localName); } - table.addTable(sb, options.stableStateTimePeriod); + table.addTable(sb, options.stableStateTimePeriod()); } private void storeNodeInfo(ContentCluster cluster, int nodeIndex, NodeType nodeType, Map<Integer, NodeInfo> nodeInfoByIndex) { @@ -200,4 +205,80 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa nodeInfoByIndex.put(nodeIndex, nodeInfo); } + public void writeHtmlState(StringBuilder sb, FleetControllerOptions options) { + String slobrokspecs = ""; + for (int i = 0; i < options.slobrokConnectionSpecs().length; ++i) { + if (i != 0) slobrokspecs += "<br>"; + slobrokspecs += options.slobrokConnectionSpecs()[i]; + } + sb.append("<h1>Current config</h1>\n") + .append("<p>Fleet controller config id: ").append(options.fleetControllerConfigId() == null ? null : options.fleetControllerConfigId().replaceAll("\n", "<br>\n")).append("</p>\n") + .append("<p>Slobrok config id: ").append(options.slobrokConfigId() == null ? null : options.slobrokConfigId().replaceAll("\n", "<br>\n")).append("</p>\n") + .append("<table border=\"1\" cellspacing=\"0\"><tr><th>Property</th><th>Value</th></tr>\n"); + + sb.append("<tr><td><nobr>Cluster name</nobr></td><td align=\"right\">").append(options.clusterName()).append("</td></tr>"); + sb.append("<tr><td><nobr>Fleet controller index</nobr></td><td align=\"right\">").append(options.fleetControllerIndex()).append("/").append(options.fleetControllerCount()).append("</td></tr>"); + sb.append("<tr><td><nobr>Number of fleetcontrollers gathering states from nodes</nobr></td><td align=\"right\">").append(options.stateGatherCount()).append("</td></tr>"); + + sb.append("<tr><td><nobr>Slobrok connection spec</nobr></td><td align=\"right\">").append(slobrokspecs).append("</td></tr>"); + sb.append("<tr><td><nobr>RPC port</nobr></td><td align=\"right\">").append(options.rpcPort() == 0 ? "Pick random available" : options.rpcPort()).append("</td></tr>"); + sb.append("<tr><td><nobr>HTTP port</nobr></td><td align=\"right\">").append(options.httpPort() == 0 ? "Pick random available" : options.httpPort()).append("</td></tr>"); + sb.append("<tr><td><nobr>Master cooldown period</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.masterZooKeeperCooldownPeriod())).append("</td></tr>"); + String zooKeeperAddress = (options.zooKeeperServerAddress() == null ? "Not using Zookeeper" : splitZooKeeperAddress(options.zooKeeperServerAddress())); + sb.append("<tr><td><nobr>Zookeeper server address</nobr></td><td align=\"right\">").append(zooKeeperAddress).append("</td></tr>"); + sb.append("<tr><td><nobr>Zookeeper session timeout</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.zooKeeperSessionTimeout())).append("</td></tr>"); + + sb.append("<tr><td><nobr>Cycle wait time</nobr></td><td align=\"right\">").append(options.cycleWaitTime()).append(" ms</td></tr>"); + sb.append("<tr><td><nobr>Minimum time before first clusterstate broadcast as master</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.minTimeBeforeFirstSystemStateBroadcast())).append("</td></tr>"); + sb.append("<tr><td><nobr>Minimum time between official cluster states</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.minTimeBetweenNewSystemStates())).append("</td></tr>"); + sb.append("<tr><td><nobr>Slobrok mirror backoff policy</nobr></td><td align=\"right\">").append(options.slobrokBackOffPolicy() == null ? "default" : "overridden").append("</td></tr>"); + + sb.append("<tr><td><nobr>Node state request timeout</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.nodeStateRequestTimeoutMS())).append("</td></tr>"); + sb.append("<tr><td><nobr>VDS 4.1 node state polling frequency</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.statePollingFrequency())).append("</td></tr>"); + sb.append("<tr><td><nobr>Maximum distributor transition time</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.maxTransitionTime().get(NodeType.DISTRIBUTOR))).append("</td></tr>"); + sb.append("<tr><td><nobr>Maximum storage transition time</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.maxTransitionTime().get(NodeType.STORAGE))).append("</td></tr>"); + sb.append("<tr><td><nobr>Maximum initialize without progress time</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.maxInitProgressTime())).append("</td></tr>"); + sb.append("<tr><td><nobr>Maximum premature crashes</nobr></td><td align=\"right\">").append(options.maxPrematureCrashes()).append("</td></tr>"); + sb.append("<tr><td><nobr>Stable state time period</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.stableStateTimePeriod())).append("</td></tr>"); + sb.append("<tr><td><nobr>Slobrok disconnect grace period</nobr></td><td align=\"right\">").append(RealTimer.printDuration(options.maxSlobrokDisconnectGracePeriod())).append("</td></tr>"); + + sb.append("<tr><td><nobr>Number of distributor nodes</nobr></td><td align=\"right\">").append(options.nodes() == null ? "Autodetect" : options.nodes().size()).append("</td></tr>"); + sb.append("<tr><td><nobr>Number of storage nodes</nobr></td><td align=\"right\">").append(options.nodes() == null ? "Autodetect" : options.nodes().size()).append("</td></tr>"); + sb.append("<tr><td><nobr>Minimum distributor nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(options.minDistributorNodesUp()).append("</td></tr>"); + sb.append("<tr><td><nobr>Minimum storage nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(options.minStorageNodesUp()).append("</td></tr>"); + sb.append("<tr><td><nobr>Minimum percentage of distributor nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(DecimalDot2.format(100 * options.minRatioOfDistributorNodesUp())).append(" %</td></tr>"); + sb.append("<tr><td><nobr>Minimum percentage of storage nodes being up for cluster to be up</nobr></td><td align=\"right\">").append(DecimalDot2.format(100 * options.minRatioOfStorageNodesUp())).append(" %</td></tr>"); + + sb.append("<tr><td><nobr>Show local cluster state changes</nobr></td><td align=\"right\">").append(options.showLocalSystemStatesInEventLog()).append("</td></tr>"); + sb.append("<tr><td><nobr>Maximum event log size</nobr></td><td align=\"right\">").append(options.eventLogMaxSize()).append("</td></tr>"); + sb.append("<tr><td><nobr>Maximum node event log size</nobr></td><td align=\"right\">").append(options.eventNodeLogMaxSize()).append("</td></tr>"); + sb.append("<tr><td><nobr>Wanted distribution bits</nobr></td><td align=\"right\">").append(options.distributionBits()).append("</td></tr>"); + sb.append("<tr><td><nobr>Max deferred task version wait time</nobr></td><td align=\"right\">").append(options.maxDeferredTaskVersionWaitTime().toMillis()).append("ms</td></tr>"); + sb.append("<tr><td><nobr>Cluster has global document types configured</nobr></td><td align=\"right\">").append(options.clusterHasGlobalDocumentTypes()).append("</td></tr>"); + sb.append("<tr><td><nobr>Enable 2-phase cluster state activation protocol</nobr></td><td align=\"right\">").append(options.enableTwoPhaseClusterStateActivation()).append("</td></tr>"); + sb.append("<tr><td><nobr>Cluster auto feed block on resource exhaustion enabled</nobr></td><td align=\"right\">") + .append(options.clusterFeedBlockEnabled()).append("</td></tr>"); + sb.append("<tr><td><nobr>Feed block limits</nobr></td><td align=\"right\">") + .append(options.clusterFeedBlockLimit().entrySet().stream() + .map(kv -> String.format("%s: %.2f%%", kv.getKey(), kv.getValue() * 100.0)) + .collect(Collectors.joining("<br/>"))).append("</td></tr>"); + + sb.append("</table>"); + } + + private static String splitZooKeeperAddress(String s) { + StringBuilder sb = new StringBuilder(); + while (true) { + int index = s.indexOf(','); + if (index > 0) { + sb.append(s.substring(0, index + 1)).append(' '); + s = s.substring(index+1); + } else { + break; + } + } + sb.append(s); + return sb.toString(); + } + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java index 69144d23fe5..79e6a91f561 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java @@ -31,7 +31,7 @@ public class StatusHandler implements HttpRequestHandler { StatusPageServer.HttpRequest request; StatusPageResponse response; - // Ensure only only one use the server at a time + // Ensure only one use the server at a time private final Object queueMonitor = new Object(); // Lock safety with fleetcontroller. Wait until completion private final Object answerMonitor = new Object(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index ec6255fd13b..39878928944 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -9,6 +9,7 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; +import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,22 +44,24 @@ public class ClusterFeedBlockTest extends FleetControllerTest { private void initialize(FleetControllerOptions options) throws Exception { List<Node> nodes = new ArrayList<>(); - for (int i = 0; i < options.nodes.size(); ++i) { + for (int i = 0; i < options.nodes().size(); ++i) { nodes.add(new Node(NodeType.STORAGE, i)); nodes.add(new Node(NodeType.DISTRIBUTOR, i)); } var context = new TestFleetControllerContext(options); communicator = new DummyCommunicator(nodes, timer); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex(), options.clusterName()); var eventLog = new EventLog(timer, metricUpdater); - var cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); + var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); var stateGatherer = new NodeStateGatherer(timer, timer, eventLog); - var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress, timer); + var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress(), timer); var stateGenerator = new StateChangeHandler(context, timer, eventLog); var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); - var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex, options.fleetControllerCount, timer, timer); - ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, null, null, communicator, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex(), options.fleetControllerCount(), timer, timer); + var status = new StatusHandler.ContainerStatusPageServer(); + ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, status, null, communicator, database, + stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); ctrl.tick(); markAllNodesAsUp(options); @@ -66,7 +69,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { } private void markAllNodesAsUp(FleetControllerOptions options) throws Exception { - for (int i = 0; i < options.nodes.size(); ++i) { + for (int i = 0; i < options.nodes().size(); ++i) { communicator.setNodeState(new Node(NodeType.STORAGE, i), State.UP, ""); communicator.setNodeState(new Node(NodeType.DISTRIBUTOR, i), State.UP, ""); } @@ -81,15 +84,13 @@ public class ClusterFeedBlockTest extends FleetControllerTest { super.tearDown(); } - private static FleetControllerOptions createOptions(Map<String, Double> feedBlockLimits, - double clusterFeedBlockNoiseLevel) { - FleetControllerOptions options = defaultOptions("mycluster"); - options.setStorageDistribution(DistributionBuilder.forFlatCluster(NODE_COUNT)); - options.nodes = new HashSet<>(DistributionBuilder.buildConfiguredNodes(NODE_COUNT)); - options.clusterFeedBlockEnabled = true; - options.clusterFeedBlockLimit = Map.copyOf(feedBlockLimits); - options.clusterFeedBlockNoiseLevel = clusterFeedBlockNoiseLevel; - return options; + private static FleetControllerOptions createOptions(Map<String, Double> feedBlockLimits, double clusterFeedBlockNoiseLevel) { + return defaultOptions("mycluster") + .setStorageDistribution(DistributionBuilder.forFlatCluster(NODE_COUNT)) + .setNodes(new HashSet<>(DistributionBuilder.buildConfiguredNodes(NODE_COUNT))) + .setClusterFeedBlockEnabled(true) + .setClusterFeedBlockLimit(feedBlockLimits) + .setClusterFeedBlockNoiseLevel(clusterFeedBlockNoiseLevel).build(); } private static FleetControllerOptions createOptions(Map<String, Double> feedBlockLimits) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java index ef1d676fc4a..30c90ee0664 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java @@ -861,23 +861,25 @@ public class ClusterStateGeneratorTest { @Test void generator_params_can_inherit_values_from_controller_options() { - FleetControllerOptions options = new FleetControllerOptions("foocluster", Set.of(new ConfiguredNode(0, false))); - options.maxPrematureCrashes = 1; - options.minStorageNodesUp = 2; - options.minDistributorNodesUp = 3; - options.minRatioOfStorageNodesUp = 0.4; - options.minRatioOfDistributorNodesUp = 0.5; - options.minNodeRatioPerGroup = 0.6; - options.distributionBits = 7; - options.maxTransitionTime = ClusterStateGenerator.Params.buildTransitionTimeMap(1000, 2000); + FleetControllerOptions options = new FleetControllerOptions.Builder("foocluster", Set.of(new ConfiguredNode(0, false))) + .setMaxPrematureCrashes(1) + .setMinStorageNodesUp(2) + .setMinDistributorNodesUp(3) + .setMinRatioOfStorageNodesUp(0.4) + .setMinRatioOfDistributorNodesUp(0.5) + .setMinNodeRatioPerGroup(0.6) + .setDistributionBits(7) + .setMaxTransitionTime(NodeType.DISTRIBUTOR, 1000) + .setMaxTransitionTime(NodeType.STORAGE, 2000).build(); + final ClusterStateGenerator.Params params = ClusterStateGenerator.Params.fromOptions(options); - assertThat(params.maxPrematureCrashes, equalTo(options.maxPrematureCrashes)); - assertThat(params.minStorageNodesUp, equalTo(options.minStorageNodesUp)); - assertThat(params.minDistributorNodesUp, equalTo(options.minDistributorNodesUp)); - assertThat(params.minRatioOfStorageNodesUp, equalTo(options.minRatioOfStorageNodesUp)); - assertThat(params.minRatioOfDistributorNodesUp, equalTo(options.minRatioOfDistributorNodesUp)); - assertThat(params.minNodeRatioPerGroup, equalTo(options.minNodeRatioPerGroup)); - assertThat(params.transitionTimes, equalTo(options.maxTransitionTime)); + assertThat(params.maxPrematureCrashes, equalTo(options.maxPrematureCrashes())); + assertThat(params.minStorageNodesUp, equalTo(options.minStorageNodesUp())); + assertThat(params.minDistributorNodesUp, equalTo(options.minDistributorNodesUp())); + assertThat(params.minRatioOfStorageNodesUp, equalTo(options.minRatioOfStorageNodesUp())); + assertThat(params.minRatioOfDistributorNodesUp, equalTo(options.minRatioOfDistributorNodesUp())); + assertThat(params.minNodeRatioPerGroup, equalTo(options.minNodeRatioPerGroup())); + assertThat(params.transitionTimes, equalTo(options.maxTransitionTime())); } @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java index 9a6a9e063ac..26bb21da3d4 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java @@ -28,10 +28,10 @@ public class DatabaseTest extends FleetControllerTest { @Test void testWantedStatesInZooKeeper() throws Exception { startingTest("DatabaseTest::testWantedStatesInZooKeeper"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.zooKeeperServerAddress = "127.0.0.1"; - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster"); + builder.setZooKeeperServerAddress("127.0.0.1"); + setUpFleetController(true, builder); + setUpVdsNodes(true); log.info("WAITING FOR STABLE SYSTEM"); waitForStableSystem(); @@ -82,12 +82,12 @@ public class DatabaseTest extends FleetControllerTest { @Test void testWantedStateOfUnknownNode() throws Exception { startingTest("DatabaseTest::testWantedStatesOfUnknownNode"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.minRatioOfDistributorNodesUp = 0; - options.minRatioOfStorageNodesUp = 0; - options.zooKeeperServerAddress = "localhost"; - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setMinRatioOfDistributorNodesUp(0) + .setMinRatioOfStorageNodesUp(0) + .setZooKeeperServerAddress("localhost"); + setUpFleetController(true, builder); + setUpVdsNodes(true); waitForStableSystem(); // Populate map of wanted states we should have diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java index d3b0addbb13..bf8bb722e3d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DistributionBitCountTest.java @@ -15,14 +15,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class DistributionBitCountTest extends FleetControllerTest { - private void setUpSystem(String testName) throws Exception { + private FleetControllerOptions setUpSystem(String testName) throws Exception { List<ConfiguredNode> configuredNodes = new ArrayList<>(); for (int i = 0 ; i < 10; i++) { configuredNodes.add(new ConfiguredNode(i, false)); } - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.distributionBits = 17; - setUpFleetController(false, options); + var builder = defaultOptions("mycluster", configuredNodes); + builder.setDistributionBits(17); + setUpFleetController(false, builder); startingTest(testName); List<DummyVdsNode> nodes = setUpVdsNodes(false, new DummyVdsNodeOptions(), true, configuredNodes); for (DummyVdsNode node : nodes) { @@ -30,6 +30,7 @@ public class DistributionBitCountTest extends FleetControllerTest { node.connect(); } waitForState("version:\\d+ bits:17 distributor:10 storage:10"); + return builder.build(); } /** @@ -38,14 +39,15 @@ public class DistributionBitCountTest extends FleetControllerTest { */ @Test void testDistributionBitCountConfigIncrease() throws Exception { - setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigIncrease"); - options.distributionBits = 20; - fleetController.updateOptions(options); + var options = setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigIncrease"); + var builder = FleetControllerOptions.Builder.copy(options); + builder.setDistributionBits(20); + fleetController.updateOptions(builder.build()); ClusterState currentState = waitForState("version:\\d+ bits:20 distributor:10 storage:10"); int version = currentState.getVersion(); - options.distributionBits = 23; - fleetController.updateOptions(options); + builder.setDistributionBits(23); + fleetController.updateOptions(builder.build()); assertEquals(version, currentState.getVersion()); } @@ -54,13 +56,13 @@ public class DistributionBitCountTest extends FleetControllerTest { */ @Test void testDistributionBitCountConfigDecrease() throws Exception { - setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigDecrease"); - options.distributionBits = 12; - fleetController.updateOptions(options); + FleetControllerOptions options = setUpSystem("DistributionBitCountTest::testDistributionBitCountConfigDecrease"); + var builder = FleetControllerOptions.Builder.copy(options); + builder.setDistributionBits(12); + fleetController.updateOptions(builder.build()); waitForState("version:\\d+ bits:12 distributor:10 storage:10"); } - /** * Test that when storage node reports higher bit count, but another storage * node has equally low bitcount, the fleetcontroller does nothing. diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index fd24966e26a..fbaf41fdd47 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -33,6 +33,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static ai.vespa.validation.Validation.requireAtLeast; + /** * * Used to fake a node in VDS, such that we can test the fleetcontroller without dummy interface for talking to @@ -100,7 +102,7 @@ public class DummyVdsNode { log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Responding to node state request at time " + currentTime); r.request.returnValues().add(new StringValue(nodeState.serialize())); if (r.request.methodName().equals("getnodestate3")) { - r.request.returnValues().add(new StringValue("No host info in dummy implementation")); + r.request.returnValues().add(new StringValue(hostInfo)); } r.request.returnRequest(); it.remove(); @@ -118,14 +120,15 @@ public class DummyVdsNode { } }; - public DummyVdsNode(Timer timer, DummyVdsNodeOptions options, String[] slobrokConnectionSpecs, String clusterName, boolean distributor, int index) throws Exception { + public DummyVdsNode(Timer timer, DummyVdsNodeOptions options, String[] slobrokConnectionSpecs, String clusterName, + NodeType nodeType, int index) throws Exception { this.timer = timer; this.slobrokConnectionSpecs = slobrokConnectionSpecs; this.clusterName = clusterName; - type = distributor ? NodeType.DISTRIBUTOR : NodeType.STORAGE; + type = nodeType; this.index = index; this.nodeState = new NodeState(type, State.UP); - this.stateCommunicationVersion = options.stateCommunicationVersion; + this.stateCommunicationVersion = requireAtLeast(options.stateCommunicationVersion, "state communication version cannot be less than 2", 2); messageResponder.start(); nodeState.setStartTimestamp(timer.getCurrentTimeInMillis() / 1000); } @@ -299,14 +302,6 @@ public class DummyVdsNode { m.returnDesc(0, "returnCode", "Returncode of request. Should be 0 = OK"); supervisor.addMethod(m); - m = new Method("getnodestate", "", "issi", this::rpc_getNodeState); - m.methodDesc("Get nodeState of a node"); - m.returnDesc(0, "returnCode", "Returncode of request. Should be 1 = OK"); - m.returnDesc(1, "returnMessage", "Textual error message if returncode is not ok."); - m.returnDesc(2, "nodeState", "The node state of the given node"); - m.returnDesc(3, "progress", "Progress in percent of node initialization"); - supervisor.addMethod(m); - m = new Method("setsystemstate", "s", "is", this::rpc_setSystemState); m.methodDesc("Set system state of entire system"); m.paramDesc(0, "systemState", "new systemstate"); @@ -362,28 +357,13 @@ public class DummyVdsNode { } } - private void rpc_getNodeState(Request req) { - synchronized(timer) { - if (!negotiatedHandle) { - req.setError(75000, "Connection not bound to a handle"); - return; - } - String stateString = nodeState.serialize(-1, true); - log.log(Level.FINE, () -> "Dummy node " + this + " got old type get node state request, answering: " + stateString); - req.returnValues().add(new Int32Value(1)); - req.returnValues().add(new StringValue("")); - req.returnValues().add(new StringValue(stateString)); - req.returnValues().add(new Int32Value(0)); - } - } - boolean sendGetNodeStateReply(int index) { for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.request.parameters().size() > 2 && r.request.parameters().get(2).asInt32() == index) { log.log(Level.FINE, () -> "Dummy node " + DummyVdsNode.this + ": Responding to node state reply from controller " + index + " as we received new one"); r.request.returnValues().add(new StringValue(nodeState.serialize())); - r.request.returnValues().add(new StringValue("No host info from dummy implementation")); + r.request.returnValues().add(new StringValue(hostInfo)); r.request.returnRequest(); it.remove(); ++outdatedStateReplies; @@ -416,7 +396,7 @@ public class DummyVdsNode { log.log(Level.FINE, () -> "Dummy node " + this + ": Request had " + (givenState == null ? "no state" : "different state(" + givenState +")") + ". Answering with " + nodeState); req.returnValues().add(new StringValue(nodeState.serialize())); if (req.methodName().equals("getnodestate3")) { - req.returnValues().add(new StringValue("Dummy node host info")); + req.returnValues().add(new StringValue(hostInfo)); } ++immediateStateReplies; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java index c391fc27813..d6b5e7fa5d2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNodeOptions.java @@ -8,6 +8,11 @@ import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; * over regular RPC. */ public class DummyVdsNodeOptions { - // 0 - 4.1, 1 - 4.2-5.0.10, 2 - 5.0.11+, 3 - 6.220+, 4 - 7.24+ + // Rpc method versions and which Vespa versions supports each version: + // 0 - 4.1 + // 1 - 4.2-5.0.10 + // 2 - 5.0.11+ + // 3 - 6.220+ + // 4 - 7.24+ int stateCommunicationVersion = RPCCommunicator.ACTIVATE_CLUSTER_STATE_VERSION_RPC_VERSION; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index c39a5c52836..58a076a6176 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -35,15 +35,15 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.yahoo.vdslib.state.NodeType.DISTRIBUTOR; +import static com.yahoo.vdslib.state.NodeType.STORAGE; import static org.junit.jupiter.api.Assertions.fail; /** @@ -111,53 +111,53 @@ public abstract class FleetControllerTest implements Waiter { testName = name; } - static protected FleetControllerOptions defaultOptions(String clusterName) { + static protected FleetControllerOptions.Builder defaultOptions(String clusterName) { return defaultOptions(clusterName, DEFAULT_NODE_COUNT); } - static protected FleetControllerOptions defaultOptions(String clusterName, int nodeCount) { + static protected FleetControllerOptions.Builder defaultOptions(String clusterName, int nodeCount) { return defaultOptions(clusterName, IntStream.range(0, nodeCount) .mapToObj(i -> new ConfiguredNode(i, false)) .collect(Collectors.toSet())); } - static protected FleetControllerOptions defaultOptions(String clusterName, Collection<ConfiguredNode> nodes) { - var opts = new FleetControllerOptions(clusterName, nodes); - opts.enableTwoPhaseClusterStateActivation = true; // Enable by default, tests can explicitly disable. - return opts; + static protected FleetControllerOptions.Builder defaultOptions(String clusterName, Collection<ConfiguredNode> nodes) { + var builder = new FleetControllerOptions.Builder(clusterName, nodes); + builder.enableTwoPhaseClusterStateActivation(true); // Enable by default, tests can explicitly disable. + return builder; } - void setUpSystem(FleetControllerOptions options) throws Exception { + void setUpSystem(FleetControllerOptions.Builder builder) throws Exception { log.log(Level.FINE, "Setting up system"); slobrok = new Slobrok(); - this.options = options; - if (options.zooKeeperServerAddress != null) { + if (builder.zooKeeperServerAddress() != null) { zooKeeperServer = new ZooKeeperTestServer(); - this.options.zooKeeperServerAddress = zooKeeperServer.getAddress(); - log.log(Level.FINE, "Set up new zookeeper server at " + this.options.zooKeeperServerAddress); + builder.setZooKeeperServerAddress(zooKeeperServer.getAddress()); + log.log(Level.FINE, "Set up new zookeeper server at " + zooKeeperServer.getAddress()); } - this.options.slobrokConnectionSpecs = getSlobrokConnectionSpecs(slobrok); + builder.setSlobrokConnectionSpecs(getSlobrokConnectionSpecs(slobrok)); + this.options = builder.build(); } FleetController createFleetController(boolean useFakeTimer, FleetControllerOptions options) throws Exception { var context = new TestFleetControllerContext(options); Timer timer = useFakeTimer ? this.timer : new RealTimer(); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex(), options.clusterName()); var log = new EventLog(timer, metricUpdater); - var cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); + var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); var stateGatherer = new NodeStateGatherer(timer, timer, log); var communicator = new RPCCommunicator( RPCCommunicator.createRealSupervisor(), timer, - options.fleetControllerIndex, - options.nodeStateRequestTimeoutMS, - options.nodeStateRequestTimeoutEarliestPercentage, - options.nodeStateRequestTimeoutLatestPercentage, - options.nodeStateRequestRoundTripTimeMaxSeconds); + options.fleetControllerIndex(), + options.nodeStateRequestTimeoutMS(), + options.nodeStateRequestTimeoutEarliestPercentage(), + options.nodeStateRequestTimeoutLatestPercentage(), + options.nodeStateRequestRoundTripTimeMaxSeconds()); var lookUp = new SlobrokClient(context, timer); lookUp.setSlobrokConnectionSpecs(new String[0]); - var rpcServer = new RpcServer(timer, timer, options.clusterName, options.fleetControllerIndex, options.slobrokBackOffPolicy); - var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress, timer); + var rpcServer = new RpcServer(timer, timer, options.clusterName(), options.fleetControllerIndex(), options.slobrokBackOffPolicy()); + var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress(), timer); // Setting this <1000 ms causes ECONNREFUSED on socket trying to connect to ZK server, in ZooKeeper, // after creating a new ZooKeeper (session). This causes ~10s extra time to connect after connection loss. @@ -166,7 +166,7 @@ public abstract class FleetControllerTest implements Waiter { var stateGenerator = new StateChangeHandler(context, timer, log); var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); - var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex, options.fleetControllerCount, timer, timer); + var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex(), options.fleetControllerCount(), timer, timer); var status = new StatusHandler.ContainerStatusPageServer(); var controller = new FleetController(context, timer, log, cluster, stateGatherer, communicator, status, rpcServer, lookUp, @@ -175,13 +175,15 @@ public abstract class FleetControllerTest implements Waiter { return controller; } - protected void setUpFleetController(boolean useFakeTimer, FleetControllerOptions options) throws Exception { - if (slobrok == null) setUpSystem(options); + protected FleetControllerOptions setUpFleetController(boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception { + if (slobrok == null) setUpSystem(builder); + options = builder.build(); if (fleetController == null) { fleetController = createFleetController(useFakeTimer, options); } else { throw new Exception("called setUpFleetcontroller but it was already setup"); } + return options; } void stopFleetController() throws Exception { @@ -199,8 +201,8 @@ public abstract class FleetControllerTest implements Waiter { } } - protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options) throws Exception { - setUpVdsNodes(useFakeTimer, options, false); + protected void setUpVdsNodes(boolean useFakeTimer) throws Exception { + setUpVdsNodes(useFakeTimer, new DummyVdsNodeOptions(), false); } protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options, boolean startDisconnected) throws Exception { setUpVdsNodes(useFakeTimer, options, startDisconnected, DEFAULT_NODE_COUNT); @@ -211,15 +213,23 @@ public abstract class FleetControllerTest implements Waiter { nodeIndexes.add(this.nodes.size()/2 + i); // divide by 2 because there are 2 nodes (storage and distributor) per index setUpVdsNodes(useFakeTimer, options, startDisconnected, nodeIndexes); } - protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options, boolean startDisconnected, Set<Integer> nodeIndexes) throws Exception { - String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); + protected void setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions nodeOptions, boolean startDisconnected, Set<Integer> nodeIndexes) throws Exception { for (int nodeIndex : nodeIndexes) { - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName, true, nodeIndex)); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName, false, nodeIndex)); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); + nodes.add(createNode(useFakeTimer, nodeOptions, startDisconnected, DISTRIBUTOR, nodeIndex)); + nodes.add(createNode(useFakeTimer, nodeOptions, startDisconnected, STORAGE, nodeIndex)); } } + + private DummyVdsNode createNode(boolean useFakeTimer, DummyVdsNodeOptions nodeOptions, boolean startDisconnected, + NodeType nodeType, int nodeIndex) throws Exception { + String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); + DummyVdsNode node = new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), nodeOptions, connectionSpecs, + options.clusterName(), nodeType, nodeIndex); + if ( ! startDisconnected) + node.connect(); + return node; + } + // TODO: Replace all usages of the above setUp methods with this one, and remove the nodes field /** @@ -228,14 +238,10 @@ public abstract class FleetControllerTest implements Waiter { * the returned list is twice as large as configuredNodes. */ protected List<DummyVdsNode> setUpVdsNodes(boolean useFakeTimer, DummyVdsNodeOptions options, boolean startDisconnected, List<ConfiguredNode> configuredNodes) throws Exception { - String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); nodes = new ArrayList<>(); - final boolean distributor = true; for (ConfiguredNode configuredNode : configuredNodes) { - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName, distributor, configuredNode.index())); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); - nodes.add(new DummyVdsNode(useFakeTimer ? timer : new RealTimer(), options, connectionSpecs, this.options.clusterName, !distributor, configuredNode.index())); - if ( ! startDisconnected) nodes.get(nodes.size() - 1).connect(); + nodes.add(createNode(useFakeTimer, options, startDisconnected, DISTRIBUTOR, configuredNode.index())); + nodes.add(createNode(useFakeTimer, options, startDisconnected, STORAGE, configuredNode.index())); } return nodes; } @@ -272,13 +278,6 @@ public abstract class FleetControllerTest implements Waiter { subsetWaiter.waitForState(expectedState); } - static Map<NodeType, Integer> transitionTimes(int milliseconds) { - Map<NodeType, Integer> maxTransitionTime = new TreeMap<>(); - maxTransitionTime.put(NodeType.DISTRIBUTOR, milliseconds); - maxTransitionTime.put(NodeType.STORAGE, milliseconds); - return maxTransitionTime; - } - protected void tearDownSystem() throws Exception { if (testName != null) { //log.log(Level.INFO, "STOPPING TEST " + testName); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java index 7f6684e595d..2d2790119e9 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownLiveConfigTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.NodeType; import org.junit.jupiter.api.Test; import java.util.HashSet; @@ -9,39 +10,39 @@ import static org.junit.jupiter.api.Assertions.assertFalse; public class GroupAutoTakedownLiveConfigTest extends FleetControllerTest { - private static FleetControllerOptions createOptions(DistributionBuilder.GroupBuilder groupBuilder, double minNodeRatio) { - FleetControllerOptions options = defaultOptions("mycluster"); - options.setStorageDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)); - options.nodes = new HashSet<>(DistributionBuilder.buildConfiguredNodes(groupBuilder.totalNodeCount())); - options.minNodeRatioPerGroup = minNodeRatio; - options.maxTransitionTime = transitionTimes(0); - return options; + private static FleetControllerOptions.Builder createOptions(DistributionBuilder.GroupBuilder groupBuilder, double minNodeRatio) { + return defaultOptions("mycluster") + .setStorageDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)) + .setNodes(new HashSet<>(DistributionBuilder.buildConfiguredNodes(groupBuilder.totalNodeCount()))) + .setMinNodeRatioPerGroup(minNodeRatio) + .setMaxTransitionTime(NodeType.DISTRIBUTOR, 0) + .setMaxTransitionTime(NodeType.STORAGE, 0); } private void updateConfigLive(FleetControllerOptions newOptions) { this.fleetController.updateOptions(newOptions); } - private void reconfigureWithMinNodeRatio(double minNodeRatio) { - FleetControllerOptions newOptions = this.options.clone(); - newOptions.minNodeRatioPerGroup = minNodeRatio; - updateConfigLive(newOptions); + private void reconfigureWithMinNodeRatio(FleetControllerOptions options, double minNodeRatio) { + FleetControllerOptions.Builder newOptions = FleetControllerOptions.Builder.copy(options); + newOptions.setMinNodeRatioPerGroup(minNodeRatio); + updateConfigLive(newOptions.build()); } - private void reconfigureWithDistribution(DistributionBuilder.GroupBuilder groupBuilder) { - FleetControllerOptions newOptions = this.options.clone(); - newOptions.nodes = new HashSet<>(DistributionBuilder.buildConfiguredNodes(groupBuilder.totalNodeCount())); - newOptions.storageDistribution = DistributionBuilder.forHierarchicCluster(groupBuilder); - updateConfigLive(newOptions); + private void reconfigureWithDistribution(FleetControllerOptions options, DistributionBuilder.GroupBuilder groupBuilder) { + FleetControllerOptions.Builder builder = + FleetControllerOptions.Builder.copy(options) + .setNodes(new HashSet<>(DistributionBuilder.buildConfiguredNodes(groupBuilder.totalNodeCount()))) + .setStorageDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)); + updateConfigLive(builder.build()); } - private void setUp3x3ClusterWithMinNodeRatio(double minNodeRatio) throws Exception { - FleetControllerOptions options = createOptions( - DistributionBuilder.withGroups(3).eachWithNodeCount(3), - minNodeRatio); + private FleetControllerOptions setUp3x3ClusterWithMinNodeRatio(double minNodeRatio) throws Exception { + FleetControllerOptions.Builder options = createOptions(DistributionBuilder.withGroups(3).eachWithNodeCount(3), minNodeRatio); setUpFleetController(true, options); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, 9); waitForState("version:\\d+ distributor:9 storage:9"); + return options.build(); } private void takeDownContentNode(int index) { @@ -62,28 +63,28 @@ public class GroupAutoTakedownLiveConfigTest extends FleetControllerTest { @Test void min_ratio_live_reconfig_immediately_takes_effect() throws Exception { // Initially, arbitrarily many nodes may be down in a group. - setUp3x3ClusterWithMinNodeRatio(0.0); + var options = setUp3x3ClusterWithMinNodeRatio(0.0); takeDownContentNode(3); waitForStateExcludingNodeSubset("version:\\d+ distributor:9 storage:9 .3.s:d", asIntSet(3)); - reconfigureWithMinNodeRatio(0.67); + reconfigureWithMinNodeRatio(options, 0.67); waitForStateExcludingNodeSubset("version:\\d+ distributor:9 storage:9 .3.s:d .4.s:d .5.s:d", asIntSet(3)); - reconfigureWithMinNodeRatio(0.0); + reconfigureWithMinNodeRatio(options, 0.0); // Aaaand back up again! waitForStateExcludingNodeSubset("version:\\d+ distributor:9 storage:9 .3.s:d", asIntSet(3)); } @Test void live_distribution_config_changes_trigger_cluster_state_change() throws Exception { - setUp3x3ClusterWithMinNodeRatio(0.65); + var options = setUp3x3ClusterWithMinNodeRatio(0.65); takeDownContentNode(6); // Not enough nodes down to trigger group take-down yet waitForStateExcludingNodeSubset("version:\\d+ distributor:9 storage:9 .6.s:d", asIntSet(6)); // Removing a node from the same group as node 6 will dip it under the configured threshold, // taking down the entire group. In this case we configure out node 8. - reconfigureWithDistribution(DistributionBuilder.withGroupNodes(3, 3, 2)); + reconfigureWithDistribution(options, DistributionBuilder.withGroupNodes(3, 3, 2)); waitForStateExcludingNodeSubset("version:\\d+ distributor:8 storage:6", asIntSet(6, 8)); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index 702a18b7bbc..c5d7c520751 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -39,31 +39,32 @@ public class MasterElectionTest extends FleetControllerTest { private static int defaultZkSessionTimeoutInMillis() { return 30_000; } - protected void setUpFleetController(int count, boolean useFakeTimer, FleetControllerOptions options) throws Exception { + protected FleetControllerOptions setUpFleetController(int count, boolean useFakeTimer, FleetControllerOptions.Builder builder) throws Exception { if (zooKeeperServer == null) { zooKeeperServer = new ZooKeeperTestServer(); } slobrok = new Slobrok(); - this.options = options; - this.options.zooKeeperSessionTimeout = defaultZkSessionTimeoutInMillis(); - this.options.zooKeeperServerAddress = zooKeeperServer.getAddress(); - this.options.slobrokConnectionSpecs = getSlobrokConnectionSpecs(slobrok); - this.options.fleetControllerCount = count; - for (int i=0; i<count; ++i) { - FleetControllerOptions nodeOptions = options.clone(); - nodeOptions.fleetControllerIndex = i; - fleetControllers.add(createFleetController(useFakeTimer, nodeOptions)); + builder.setZooKeeperSessionTimeout(defaultZkSessionTimeoutInMillis()) + .setZooKeeperServerAddress(zooKeeperServer.getAddress()) + .setSlobrokConnectionSpecs(getSlobrokConnectionSpecs(slobrok)) + .setCount(count); + options = builder.build(); + for (int i = 0; i < count; ++i) { + FleetControllerOptions.Builder b = FleetControllerOptions.Builder.copy(options); + b.setIndex(i); + fleetControllers.add(createFleetController(useFakeTimer, b.build())); } + return options; } - private FleetControllerOptions adjustConfig(FleetControllerOptions o, int fleetControllerIndex, int fleetControllerCount) { - FleetControllerOptions options = o.clone(); - options.zooKeeperSessionTimeout = defaultZkSessionTimeoutInMillis(); - options.zooKeeperServerAddress = zooKeeperServer.getAddress(); - options.slobrokConnectionSpecs = getSlobrokConnectionSpecs(slobrok); - options.fleetControllerIndex = fleetControllerIndex; - options.fleetControllerCount = fleetControllerCount; - return options; + private FleetControllerOptions adjustConfig(FleetControllerOptions options, int fleetControllerIndex, int fleetControllerCount) { + return FleetControllerOptions.Builder.copy(options) + .setZooKeeperSessionTimeout(defaultZkSessionTimeoutInMillis()) + .setZooKeeperServerAddress(zooKeeperServer.getAddress()) + .setSlobrokConnectionSpecs(getSlobrokConnectionSpecs(slobrok)) + .setIndex(fleetControllerIndex) + .setCount(fleetControllerCount) + .build(); } private void waitForZookeeperDisconnected() throws TimeoutException { @@ -111,10 +112,10 @@ public class MasterElectionTest extends FleetControllerTest { void testMasterElection() throws Exception { startingTest("MasterElectionTest::testMasterElection"); log.log(Level.INFO, "STARTING TEST: MasterElectionTest::testMasterElection()"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 100; + FleetControllerOptions.Builder builder = defaultOptions("mycluster"); + builder.setMasterZooKeeperCooldownPeriod(100); boolean usingFakeTimer = false; - setUpFleetController(5, usingFakeTimer, options); + setUpFleetController(5, usingFakeTimer, builder); waitForMaster(0); log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); @@ -216,12 +217,12 @@ public class MasterElectionTest extends FleetControllerTest { @Test void testClusterStateVersionIncreasesAcrossMasterElections() throws Exception { startingTest("MasterElectionTest::testClusterStateVersionIncreasesAcrossMasterElections"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 1; + FleetControllerOptions.Builder options = defaultOptions("mycluster"); + options.setMasterZooKeeperCooldownPeriod(1); setUpFleetController(3, false, options); // Currently need to have content nodes present for the cluster controller to even bother // attempting to persisting its cluster state version to ZK. - setUpVdsNodes(false, new DummyVdsNodeOptions()); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForStableSystem(); waitForMaster(0); @@ -237,10 +238,10 @@ public class MasterElectionTest extends FleetControllerTest { @Test void testVotingCorrectnessInFaceOfZKDisconnect() throws Exception { startingTest("MasterElectionTest::testVotingCorrectnessInFaceOfZKDisconnect"); - FleetControllerOptions options = defaultOptions("mycluster"); + FleetControllerOptions.Builder options = defaultOptions("mycluster"); // "Magic" port value is in range allocated to module for testing. zooKeeperServer = ZooKeeperTestServer.createWithFixedPort(18342); - options.masterZooKeeperCooldownPeriod = 100; + options.setMasterZooKeeperCooldownPeriod(100); setUpFleetController(2, false, options); waitForMaster(0); @@ -258,10 +259,10 @@ public class MasterElectionTest extends FleetControllerTest { @Test void testZooKeeperUnavailable() throws Exception { startingTest("MasterElectionTest::testZooKeeperUnavailable"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 100; - options.zooKeeperServerAddress = "localhost"; - setUpFleetController(3, false, options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setMasterZooKeeperCooldownPeriod(100) + .setZooKeeperServerAddress("localhost"); + setUpFleetController(3, false, builder); waitForMaster(0); log.log(Level.INFO, "STOPPING ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); @@ -277,10 +278,11 @@ public class MasterElectionTest extends FleetControllerTest { zooKeeperServer = new ZooKeeperTestServer(); log.log(Level.INFO, "STARTED ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); for (FleetController fc : fleetControllers) { - FleetControllerOptions myoptions = fc.getOptions(); - myoptions.zooKeeperServerAddress = zooKeeperServer.getAddress(); - fc.updateOptions(myoptions); - log.log(Level.INFO, "Should now have sent out new zookeeper server address " + myoptions.zooKeeperServerAddress + " to fleetcontroller " + myoptions.fleetControllerIndex); + FleetControllerOptions.Builder myoptions = FleetControllerOptions.Builder.copy(fc.getOptions()); + myoptions.setZooKeeperServerAddress(zooKeeperServer.getAddress()); + fc.updateOptions(myoptions.build()); + log.log(Level.INFO, "Should now have sent out new zookeeper server address " + myoptions.zooKeeperServerAddress() + + " to fleetcontroller " + myoptions.fleetControllerIndex()); } waitForMaster(0); log.log(Level.INFO, "SHUTTING DOWN"); @@ -290,8 +292,8 @@ public class MasterElectionTest extends FleetControllerTest { @Disabled("Unstable, disable test, as functionality is not deemed critical") void testMasterZooKeeperCooldown() throws Exception { startingTest("MasterElectionTest::testMasterZooKeeperCooldown"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 3600 * 1000; // An hour + FleetControllerOptions.Builder options = defaultOptions("mycluster"); + options.setMasterZooKeeperCooldownPeriod(3600 * 1000); // An hour setUpFleetController(3, true, options); waitForMaster(0); timer.advanceTime(24 * 3600 * 1000); // A day @@ -338,8 +340,8 @@ public class MasterElectionTest extends FleetControllerTest { @Test void testGetMaster() throws Exception { startingTest("MasterElectionTest::testGetMaster"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 3600 * 1000; // An hour + FleetControllerOptions.Builder options = defaultOptions("mycluster"); + options.setMasterZooKeeperCooldownPeriod(3600 * 1000); // An hour setUpFleetController(3, true, options); waitForMaster(0); @@ -420,12 +422,12 @@ public class MasterElectionTest extends FleetControllerTest { @Test void testReconfigure() throws Exception { startingTest("MasterElectionTest::testReconfigure"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 1; + FleetControllerOptions.Builder options = defaultOptions("mycluster"); + options.setMasterZooKeeperCooldownPeriod(1); setUpFleetController(3, false, options); waitForMaster(0); - FleetControllerOptions newOptions = options.clone(); + FleetControllerOptions newOptions = FleetControllerOptions.Builder.copy(options.build()).build(); for (int i = 0; i < fleetControllers.size(); ++i) { FleetControllerOptions nodeOptions = adjustConfig(newOptions, i, fleetControllers.size()); fleetControllers.get(i).updateOptions(nodeOptions); @@ -445,14 +447,14 @@ public class MasterElectionTest extends FleetControllerTest { @Test void cluster_state_version_written_to_zookeeper_even_with_empty_send_set() throws Exception { startingTest("MasterElectionTest::cluster_state_version_written_to_zookeeper_even_with_empty_send_set"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 1; - options.minRatioOfDistributorNodesUp = 0; - options.minRatioOfStorageNodesUp = 0; - options.minDistributorNodesUp = 0; - options.minStorageNodesUp = 1; - setUpFleetController(3, false, options); - setUpVdsNodes(false, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setMasterZooKeeperCooldownPeriod(1) + .setMinRatioOfDistributorNodesUp(0) + .setMinRatioOfStorageNodesUp(0) + .setMinDistributorNodesUp(0) + .setMinStorageNodesUp(1); + setUpFleetController(3, false, builder); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForStableSystem(); waitForMaster(0); @@ -491,13 +493,13 @@ public class MasterElectionTest extends FleetControllerTest { @Test void previously_published_state_is_taken_into_account_for_default_space_when_controller_bootstraps() throws Exception { startingTest("MasterElectionTest::previously_published_state_is_taken_into_account_for_default_space_when_controller_bootstraps"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.clusterHasGlobalDocumentTypes = true; - options.masterZooKeeperCooldownPeriod = 1; - options.minTimeBeforeFirstSystemStateBroadcast = 100000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setClusterHasGlobalDocumentTypes(true) + .setMasterZooKeeperCooldownPeriod(1) + .setMinTimeBeforeFirstSystemStateBroadcast(100000); boolean useFakeTimer = false; - setUpFleetController(3, useFakeTimer, options); - setUpVdsNodes(false, new DummyVdsNodeOptions()); + setUpFleetController(3, useFakeTimer, builder); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForMaster(0); waitForStableSystem(); @@ -535,12 +537,12 @@ public class MasterElectionTest extends FleetControllerTest { @Test void default_space_nodes_not_marked_as_maintenance_when_cluster_has_no_global_document_types() throws Exception { startingTest("MasterElectionTest::default_space_nodes_not_marked_as_maintenance_when_cluster_has_no_global_document_types"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.clusterHasGlobalDocumentTypes = false; - options.masterZooKeeperCooldownPeriod = 1; - options.minTimeBeforeFirstSystemStateBroadcast = 100000; - setUpFleetController(3, false, options); - setUpVdsNodes(false, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setClusterHasGlobalDocumentTypes(false) + .setMasterZooKeeperCooldownPeriod(1) + .setMinTimeBeforeFirstSystemStateBroadcast(100000); + setUpFleetController(3, false, builder); + setUpVdsNodes(false); fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing waitForMaster(0); waitForStableSystem(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java index 62a49ecf969..483c92d4861 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NoZooKeeperTest.java @@ -10,10 +10,10 @@ public class NoZooKeeperTest extends FleetControllerTest { @Test void testWantedStatesInZooKeeper() throws Exception { startingTest("NoZooKeeperTest::testWantedStatesInZooKeeper"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.zooKeeperServerAddress = null; - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + // Null is the default for zooKeeperServerAddress + FleetControllerOptions.Builder builder = defaultOptions("mycluster"); + setUpFleetController(true, builder); + setUpVdsNodes(true); waitForStableSystem(); assertTrue(nodes.get(0).isDistributor()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java index 69ddf8e2c02..8a16efe0bfa 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeSlobrokConfigurationMembershipTest.java @@ -2,50 +2,55 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.distribution.ConfiguredNode; +import com.yahoo.vdslib.state.NodeType; import org.junit.jupiter.api.Test; - +import org.junit.jupiter.api.Timeout; import java.util.Set; import java.util.TreeSet; import static org.junit.jupiter.api.Assertions.assertTrue; +@Timeout(30) public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest { private final Set<Integer> nodeIndices = asIntSet(0, 1, 2, 3); - private final int foreignNode = 6; + private final int foreignNodeIndex = 6; - private void setUpClusterWithForeignNode(Set<Integer> validIndices, final int foreignNodeIndex) throws Exception { - final Set<ConfiguredNode> configuredNodes = asConfiguredNodes(validIndices); - FleetControllerOptions options = optionsForConfiguredNodes(configuredNodes); + private FleetControllerOptions setUpClusterWithForeignNode(Set<Integer> validIndices) throws Exception { + Set<ConfiguredNode> configuredNodes = asConfiguredNodes(validIndices); + FleetControllerOptions.Builder options = optionsForConfiguredNodes(configuredNodes); setUpFleetController(true, options); Set<Integer> nodesWithStranger = new TreeSet<>(validIndices); nodesWithStranger.add(foreignNodeIndex); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, nodesWithStranger); + return options.build(); } - private FleetControllerOptions optionsForConfiguredNodes(Set<ConfiguredNode> configuredNodes) { - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.maxSlobrokDisconnectGracePeriod = 60 * 1000; - options.nodeStateRequestTimeoutMS = 10000 * 60 * 1000; - options.maxTransitionTime = transitionTimes(0); - return options; + private FleetControllerOptions.Builder optionsForConfiguredNodes(Set<ConfiguredNode> configuredNodes) { + return defaultOptions("mycluster", configuredNodes) + .setMaxSlobrokDisconnectGracePeriod(60 * 1000) + .setNodeStateRequestTimeoutMS(10000 * 60 * 1000) + .setMaxTransitionTime(NodeType.DISTRIBUTOR, 0) + .setMaxTransitionTime(NodeType.STORAGE, 0); } @Test void testSlobrokNodeOutsideConfiguredIndexSetIsNotIncludedInCluster() throws Exception { - setUpClusterWithForeignNode(nodeIndices, foreignNode); - waitForStateExcludingNodeSubset("version:\\d+ distributor:4 storage:4", asIntSet(foreignNode)); + setUpClusterWithForeignNode(nodeIndices); + waitForStateExcludingNodeSubset("version:\\d+ distributor:4 storage:4", asIntSet(foreignNodeIndex)); } @Test void testNodeSetReconfigurationForcesFreshSlobrokFetch() throws Exception { - setUpClusterWithForeignNode(nodeIndices, foreignNode); - waitForStateExcludingNodeSubset("version:\\d+ distributor:4 storage:4", asIntSet(foreignNode)); + var options = setUpClusterWithForeignNode(nodeIndices); + waitForStateExcludingNodeSubset("version:\\d+ distributor:4 storage:4", asIntSet(foreignNodeIndex)); // If we get a configuration with the node present, we have to accept it into // cluster. If we do not re-fetch state from slobrok we risk racing - nodeIndices.add(foreignNode); - options.nodes = asConfiguredNodes(nodeIndices); + nodeIndices.add(foreignNodeIndex); + var a = FleetControllerOptions.Builder.copy(options); + a.setNodes(asConfiguredNodes(nodeIndices)); + options = a.build(); fleetController.updateOptions(options); // Need to treat cluster as having 6 nodes due to ideal state algo semantics. // Note that we do not use subsetWaiter here since we want node 6 included. @@ -54,9 +59,9 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest @Test void test_removed_retired_node_is_not_included_in_state() throws Exception { - final Set<ConfiguredNode> configuredNodes = asConfiguredNodes(nodeIndices); - FleetControllerOptions options = optionsForConfiguredNodes(configuredNodes); - setUpFleetController(true, options); + Set<ConfiguredNode> configuredNodes = asConfiguredNodes(nodeIndices); + FleetControllerOptions.Builder builder = optionsForConfiguredNodes(configuredNodes); + options = setUpFleetController(true, builder); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, nodeIndices); waitForState("version:\\d+ distributor:4 storage:4"); @@ -64,13 +69,19 @@ public class NodeSlobrokConfigurationMembershipTest extends FleetControllerTest // Update options with 1 node config-retired assertTrue(configuredNodes.remove(new ConfiguredNode(0, false))); configuredNodes.add(new ConfiguredNode(0, true)); - options.nodes = configuredNodes; + + builder = FleetControllerOptions.Builder.copy(options); + builder.setNodes(configuredNodes); + options = builder.build(); fleetController.updateOptions(options); waitForState("version:\\d+ distributor:4 storage:4 .0.s:r"); // Now remove the retired node entirely from config assertTrue(configuredNodes.remove(new ConfiguredNode(0, true))); + builder = FleetControllerOptions.Builder.copy(options); + builder.setNodes(configuredNodes); + options = builder.build(); fleetController.updateOptions(options); // The previously retired node should now be marked as down, as it no longer diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index 789c86bd6cf..d36486d2eb6 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -84,7 +84,7 @@ public class RpcServerTest extends FleetControllerTest { try { startingTest("RpcServerTest::testFailOccasionallyAndIgnoreToSeeIfOtherTestsThenWork"); setUpFleetController(true, defaultOptions("mycluster")); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); } catch (Throwable t) { } @@ -94,9 +94,9 @@ public class RpcServerTest extends FleetControllerTest { void testGetSystemState() throws Exception { LogFormatter.initializeLogging(); startingTest("RpcServerTest::testGetSystemState"); - FleetControllerOptions options = defaultOptions("mycluster"); + FleetControllerOptions.Builder options = defaultOptions("mycluster"); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); assertTrue(nodes.get(0).isDistributor()); @@ -104,7 +104,7 @@ public class RpcServerTest extends FleetControllerTest { nodes.get(0).disconnect(); nodes.get(19).disconnect(); fleetController.waitForNodesInSlobrok(9, 9, timeout()); - timer.advanceTime(options.nodeStateRequestTimeoutMS + options.maxSlobrokDisconnectGracePeriod); + timer.advanceTime(options.nodeStateRequestTimeoutMS() + options.maxSlobrokDisconnectGracePeriod()); wait(new WaitCondition.StateWait(fleetController, fleetController.getMonitor()) { @Override @@ -161,12 +161,12 @@ public class RpcServerTest extends FleetControllerTest { Set<ConfiguredNode> configuredNodes = new TreeSet<>(); for (int i = 0; i < 10; i++) configuredNodes.add(new ConfiguredNode(i, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.minRatioOfStorageNodesUp = 0; - options.maxInitProgressTime = 30000; - options.stableStateTimePeriod = 60000; - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes); + builder.setMinRatioOfStorageNodesUp(0); + builder.setMaxInitProgressTime(30000); + builder.setStableStateTimePeriod(60000); + setUpFleetController(true, builder); + setUpVdsNodes(true); waitForStableSystem(); setWantedNodeState(State.DOWN, NodeType.DISTRIBUTOR, 2); @@ -256,11 +256,11 @@ public class RpcServerTest extends FleetControllerTest { for (int i = 0; i < 4; i++) configuredNodes.add(new ConfiguredNode(i, false)); configuredNodes.add(new ConfiguredNode(4, true)); // Last node is configured retired - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.minRatioOfStorageNodesUp = 0; - options.maxInitProgressTime = 30000; - options.stableStateTimePeriod = 60000; - setUpFleetController(true, options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setMinRatioOfStorageNodesUp(0) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + setUpFleetController(true, builder); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, configuredNodes); waitForState("version:\\d+ distributor:5 storage:5 .4.s:r"); @@ -291,10 +291,10 @@ public class RpcServerTest extends FleetControllerTest { List<ConfiguredNode> configuredNodes = new ArrayList<>(); for (int i = 0; i < 5; i++) configuredNodes.add(new ConfiguredNode(i, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.maxInitProgressTime = 30000; - options.stableStateTimePeriod = 60000; - setUpFleetController(true, options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + setUpFleetController(true, builder); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, configuredNodes); waitForState("version:\\d+ distributor:5 storage:5"); } @@ -315,11 +315,11 @@ public class RpcServerTest extends FleetControllerTest { configuredNodes.add(new ConfiguredNode(i, true)); configuredNodes.add(new ConfiguredNode(5, false)); configuredNodes.add(new ConfiguredNode(6, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.slobrokConnectionSpecs = this.options.slobrokConnectionSpecs; - this.options.maxInitProgressTime = 30000; - this.options.stableStateTimePeriod = 60000; - fleetController.updateOptions(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setSlobrokConnectionSpecs(this.options.slobrokConnectionSpecs()) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + fleetController.updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:m .1.s:m .2.s:r .3.s:r .4.s:r"); } @@ -345,11 +345,11 @@ public class RpcServerTest extends FleetControllerTest { Set<ConfiguredNode> configuredNodes = new TreeSet<>(); for (int i = 0; i < 7; i++) configuredNodes.add(new ConfiguredNode(i, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.slobrokConnectionSpecs = this.options.slobrokConnectionSpecs; - this.options.maxInitProgressTime = 30000; - this.options.stableStateTimePeriod = 60000; - fleetController.updateOptions(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setSlobrokConnectionSpecs(this.options.slobrokConnectionSpecs()) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + fleetController.updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:m .1.s:m"); } @@ -372,10 +372,11 @@ public class RpcServerTest extends FleetControllerTest { List<ConfiguredNode> configuredNodes = new ArrayList<>(); for (int i = 0; i < 5; i++) configuredNodes.add(new ConfiguredNode(i, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.maxInitProgressTime = 30000; - options.stableStateTimePeriod = 60000; - setUpFleetController(true, options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + options = builder.build(); + setUpFleetController(true, builder); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, configuredNodes); waitForState("version:\\d+ distributor:5 storage:5"); } @@ -384,11 +385,11 @@ public class RpcServerTest extends FleetControllerTest { Set<ConfiguredNode> configuredNodes = new TreeSet<>(); for (int i = 0; i < 5; i++) configuredNodes.add(new ConfiguredNode(i, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.slobrokConnectionSpecs = this.options.slobrokConnectionSpecs; - this.options.maxInitProgressTime = 30000; - this.options.stableStateTimePeriod = 60000; - fleetController.updateOptions(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + fleetController.updateOptions(builder.build()); waitForState("version:\\d+ distributor:5 storage:5"); } @@ -399,11 +400,11 @@ public class RpcServerTest extends FleetControllerTest { configuredNodes.add(new ConfiguredNode(i, true)); configuredNodes.add(new ConfiguredNode(5, false)); configuredNodes.add(new ConfiguredNode(6, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.slobrokConnectionSpecs = this.options.slobrokConnectionSpecs; - this.options.maxInitProgressTime = 30000; - this.options.stableStateTimePeriod = 60000; - fleetController.updateOptions(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + fleetController.updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:r .1.s:r .2.s:r .3.s:r .4.s:r"); } @@ -413,11 +414,11 @@ public class RpcServerTest extends FleetControllerTest { configuredNodes.add(new ConfiguredNode(i, true)); configuredNodes.add(new ConfiguredNode(5, false)); configuredNodes.add(new ConfiguredNode(6, false)); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); - options.slobrokConnectionSpecs = this.options.slobrokConnectionSpecs; - this.options.maxInitProgressTime = 30000; - this.options.stableStateTimePeriod = 60000; - fleetController.updateOptions(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", configuredNodes) + .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) + .setMaxInitProgressTime(30000) + .setStableStateTimePeriod(60000); + fleetController.updateOptions(builder.build()); waitForState("version:\\d+ distributor:7 storage:7 .0.s:r .1.s:r .2.s:r .3.s:r .4.s:r"); } @@ -428,10 +429,10 @@ public class RpcServerTest extends FleetControllerTest { Set<ConfiguredNode> configuredNodes = new TreeSet<>(); configuredNodes.add(new ConfiguredNode(5, false)); configuredNodes.add(new ConfiguredNode(6, false)); - FleetControllerOptions options = new FleetControllerOptions("mycluster", configuredNodes); - options.slobrokConnectionSpecs = this.options.slobrokConnectionSpecs; - this.options.maxInitProgressTimeMs = 30000; - this.options.stableStateTimePeriod = 60000; + FleetControllerOptions.Builder builder = new FleetControllerOptions.Builder("mycluster", configuredNodes) + .setSlobrokConnectionSpecs(options.slobrokConnectionSpecs()) + .setMaxInitProgressTimeMs(30000) + .setStableStateTimePeriod(60000); fleetController.updateOptions(options, 0); for (int i = 0; i < 5*2; i++) { nodes.get(i).disconnectSlobrok(); @@ -447,7 +448,7 @@ public class RpcServerTest extends FleetControllerTest { startingTest("RpcServerTest::testSetNodeState"); Set<Integer> nodeIndexes = new TreeSet<>(List.of(4, 6, 9, 10, 14, 16, 21, 22, 23, 25)); Set<ConfiguredNode> configuredNodes = nodeIndexes.stream().map(i -> new ConfiguredNode(i, false)).collect(Collectors.toSet()); - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); + FleetControllerOptions.Builder options = defaultOptions("mycluster", configuredNodes); //options.setStorageDistribution(new Distribution(getDistConfig(nodeIndexes))); setUpFleetController(true, options); setUpVdsNodes(true, new DummyVdsNodeOptions(), false, nodeIndexes); @@ -486,10 +487,10 @@ public class RpcServerTest extends FleetControllerTest { @Test void testSetNodeStateOutOfRange() throws Exception { startingTest("RpcServerTest::testSetNodeStateOutOfRange"); - FleetControllerOptions options = defaultOptions("mycluster"); + FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(2, 10))); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); int rpcPort = fleetController.getRpcPort(); @@ -514,10 +515,10 @@ public class RpcServerTest extends FleetControllerTest { @Test void testGetMaster() throws Exception { startingTest("RpcServerTest::testGetMaster"); - FleetControllerOptions options = defaultOptions("mycluster"); + FleetControllerOptions.Builder options = defaultOptions("mycluster"); options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(2, 10))); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); int rpcPort = fleetController.getRpcPort(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcVersionAutoDowngradeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcVersionAutoDowngradeTest.java index df0b873e25b..2c68c498e68 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcVersionAutoDowngradeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcVersionAutoDowngradeTest.java @@ -16,7 +16,7 @@ public class RpcVersionAutoDowngradeTest extends FleetControllerTest { for (int i = 0 ; i < 10; i++) { configuredNodes.add(new ConfiguredNode(i, false)); } - FleetControllerOptions options = defaultOptions("mycluster", configuredNodes); + FleetControllerOptions.Builder options = defaultOptions("mycluster", configuredNodes); setUpFleetController(false, options); DummyVdsNodeOptions nodeOptions = new DummyVdsNodeOptions(); nodeOptions.stateCommunicationVersion = nodeRpcVersion; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java index 78576f9600c..4cf69b778a2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java @@ -16,11 +16,11 @@ public class SlobrokTest extends FleetControllerTest { @Test void testSingleSlobrokRestart() throws Exception { startingTest("SlobrokTest::testSingleSlobrokRestart"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.nodeStateRequestTimeoutMS = 60 * 60 * 1000; - options.maxSlobrokDisconnectGracePeriod = 60 * 60 * 1000; - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setNodeStateRequestTimeoutMS(60 * 60 * 1000) + .setMaxSlobrokDisconnectGracePeriod(60 * 60 * 1000); + setUpFleetController(true, builder); + setUpVdsNodes(true); waitForStableSystem(); int version = fleetController.getSystemState().getVersion(); @@ -70,11 +70,11 @@ public class SlobrokTest extends FleetControllerTest { @Test void testNodeTooLongOutOfSlobrok() throws Exception { startingTest("SlobrokTest::testNodeTooLongOutOfSlobrok"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.maxSlobrokDisconnectGracePeriod = 60 * 1000; - options.nodeStateRequestTimeoutMS = 10000 * 60 * 1000; - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setMaxSlobrokDisconnectGracePeriod(60 * 1000) + .setNodeStateRequestTimeoutMS(10000 * 60 * 1000); + setUpFleetController(true, builder); + setUpVdsNodes(true); waitForStableSystem(); int version = fleetController.getSystemState().getVersion(); @@ -95,7 +95,7 @@ public class SlobrokTest extends FleetControllerTest { log.log(Level.INFO, "JUMPING TIME. NODE SHOULD BE MARKED DOWN"); // At this point the fleetcontroller might not have noticed that the node is out of slobrok yet. // Thus we keep advancing time another minute such that it should get down. - timer.advanceTime(options.nodeStateRequestTimeoutMS + options.maxSlobrokDisconnectGracePeriod); + timer.advanceTime(builder.nodeStateRequestTimeoutMS() + builder.maxSlobrokDisconnectGracePeriod()); waitForState("version:\\d+ distributor:10 .0.s:d storage:10"); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 426ac42fe9e..56fcd4666e8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -11,6 +11,7 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; +import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.BeforeEach; @@ -42,31 +43,33 @@ public class StateChangeTest extends FleetControllerTest { private void initialize(FleetControllerOptions options) throws Exception { List<Node> nodes = new ArrayList<>(); - for (int i = 0; i < options.nodes.size(); ++i) { + for (int i = 0; i < options.nodes().size(); ++i) { nodes.add(new Node(NodeType.STORAGE, i)); nodes.add(new Node(NodeType.DISTRIBUTOR, i)); } var context = new TestFleetControllerContext(options); communicator = new DummyCommunicator(nodes, timer); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex(), options.clusterName()); eventLog = new EventLog(timer, metricUpdater); - var cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); + var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); var stateGatherer = new NodeStateGatherer(timer, timer, eventLog); - var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress, timer); + var database = new DatabaseHandler(context, new ZooKeeperDatabaseFactory(context), timer, options.zooKeeperServerAddress(), timer); var stateGenerator = new StateChangeHandler(context, timer, eventLog); var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); - var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex, options.fleetControllerCount, timer, timer); - ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, null, null, communicator, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex(), options.fleetControllerCount(), timer, timer); + var status = new StatusHandler.ContainerStatusPageServer(); + ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, status, null, communicator, database, + stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); ctrl.tick(); - if (options.fleetControllerCount == 1) { + if (options.fleetControllerCount() == 1) { markAllNodesAsUp(options); } } private void markAllNodesAsUp(FleetControllerOptions options) throws Exception { - for (int i = 0; i < options.nodes.size(); ++i) { + for (int i = 0; i < options.nodes().size(); ++i) { communicator.setNodeState(new Node(NodeType.STORAGE, i), State.UP, ""); communicator.setNodeState(new Node(NodeType.DISTRIBUTOR, i), State.UP, ""); } @@ -100,10 +103,10 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNormalStartup() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxInitProgressTime = 50000; + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + options.setMaxInitProgressTime(50000); - initialize(options); + initialize(options.build()); // Should now pick up previous node states ctrl.tick(); @@ -114,7 +117,7 @@ public class StateChangeTest extends FleetControllerTest { } for (int i = 0; i < 100; i += 10) { - timer.advanceTime(options.maxInitProgressTime / 20); + timer.advanceTime(options.maxInitProgressTime() / 20); ctrl.tick(); for (int j = 0; j < 10; ++j) { communicator.setNodeState(new Node(NodeType.STORAGE, j), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(i / 100.0f), ""); @@ -132,21 +135,21 @@ public class StateChangeTest extends FleetControllerTest { ".4.s:i .4.i:0.1 .5.s:i .5.i:0.1 .6.s:i .6.i:0.1 .7.s:i .7.i:0.1 .8.s:i .8.i:0.1 .9.s:i .9.i:0.1", ctrl.consolidatedClusterState().toString()); - timer.advanceTime(options.maxInitProgressTime / 20); + timer.advanceTime(options.maxInitProgressTime() / 20); ctrl.tick(); for (int i = 0; i < 10; ++i) { communicator.setNodeState(new Node(NodeType.STORAGE, i), new NodeState(NodeType.STORAGE, State.UP), ""); } - timer.advanceTime(options.maxInitProgressTime / 20); + timer.advanceTime(options.maxInitProgressTime() / 20); ctrl.tick(); for (int i = 0; i < 10; ++i) { communicator.setNodeState(new Node(NodeType.DISTRIBUTOR, i), new NodeState(NodeType.STORAGE, State.UP), ""); } - timer.advanceTime(options.maxInitProgressTime / 20); + timer.advanceTime(options.maxInitProgressTime() / 20); ctrl.tick(); assertEquals("version:8 distributor:10 storage:10", ctrl.getSystemState().toString()); @@ -172,15 +175,15 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNodeGoingDownAndUp() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.nodeStateRequestTimeoutMS = 60 * 60 * 1000; - options.minTimeBetweenNewSystemStates = 0; - options.maxInitProgressTime = 50000; - // This test makes very specific assumptions about the amount of work done in a single tick. - // Two-phase cluster state activation changes this quite a bit, so disable it. At least for now. - options.enableTwoPhaseClusterStateActivation = false; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setNodeStateRequestTimeoutMS(60 * 60 * 1000) + .setMinTimeBetweenNewSystemStates(0) + .setMaxInitProgressTime(50000) + // This test makes very specific assumptions about the amount of work done in a single tick. + // Two-phase cluster state activation changes this quite a bit, so disable it. At least for now. + .enableTwoPhaseClusterStateActivation(false); - initialize(options); + initialize(builder.build()); ctrl.tick(); @@ -208,7 +211,7 @@ public class StateChangeTest extends FleetControllerTest { desc = ctrl.getReportedNodeState(new Node(NodeType.STORAGE, 0)).getDescription(); assertTrue(desc.contains("Closed at other end"), desc); - timer.advanceTime(options.maxTransitionTime.get(NodeType.STORAGE) + 1); + timer.advanceTime(builder.maxTransitionTime().get(NodeType.STORAGE) + 1); ctrl.tick(); @@ -262,15 +265,15 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNodeGoingDownAndUpNotifying() throws Exception { // Same test as above, but node manages to notify why it is going down first. - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.nodeStateRequestTimeoutMS = 60 * 60 * 1000; - options.maxSlobrokDisconnectGracePeriod = 100000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setNodeStateRequestTimeoutMS(60 * 60 * 1000) + .setMaxSlobrokDisconnectGracePeriod(100000); - initialize(options); + initialize(builder.build()); ctrl.tick(); - tick((int) options.stableStateTimePeriod + 1); + tick((int) builder.stableStateTimePeriod() + 1); communicator.setNodeState(new Node(NodeType.DISTRIBUTOR, 0), State.DOWN, "controlled shutdown"); @@ -294,7 +297,7 @@ public class StateChangeTest extends FleetControllerTest { assertTrue(desc.contains("Received signal 15 (SIGTERM - Termination signal)") || desc.contains("controlled shutdown"), desc); - tick(options.maxTransitionTime.get(NodeType.STORAGE) + 1); + tick(builder.maxTransitionTime().get(NodeType.STORAGE) + 1); assertEquals("version:6 distributor:10 storage:10 .0.s:d", ctrl.getSystemState().toString()); desc = ctrl.getReportedNodeState(new Node(NodeType.STORAGE, 0)).getDescription(); @@ -333,10 +336,10 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNodeGoingDownAndUpFast() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxSlobrokDisconnectGracePeriod = 60 * 1000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxSlobrokDisconnectGracePeriod(60 * 1000); - initialize(options); + initialize(builder.build()); ctrl.tick(); @@ -374,10 +377,10 @@ public class StateChangeTest extends FleetControllerTest { @Test void testMaintenanceWhileNormalStorageNodeRestart() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxSlobrokDisconnectGracePeriod = 60 * 1000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxSlobrokDisconnectGracePeriod(60 * 1000); - initialize(options); + initialize(builder.build()); communicator.setNodeState(new Node(NodeType.STORAGE, 6), State.DOWN, "Connection error: Closed at other end"); @@ -434,10 +437,10 @@ public class StateChangeTest extends FleetControllerTest { nodes.add(new ConfiguredNode(i, retired)); } - FleetControllerOptions options = defaultOptions("mycluster", nodes); - options.maxSlobrokDisconnectGracePeriod = 60 * 1000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", nodes) + .setMaxSlobrokDisconnectGracePeriod(60 * 1000); - initialize(options); + initialize(builder.build()); communicator.setNodeState(new Node(NodeType.STORAGE, 6), State.DOWN, "Connection error: Closed at other end"); @@ -493,10 +496,9 @@ public class StateChangeTest extends FleetControllerTest { nodes.add(new ConfiguredNode(i, retired)); } - FleetControllerOptions options = defaultOptions("mycluster", nodes); - options.maxSlobrokDisconnectGracePeriod = 60 * 1000; - - initialize(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", nodes) + .setMaxSlobrokDisconnectGracePeriod(60 * 1000); + initialize(builder.build()); communicator.setNodeState(new Node(NodeType.STORAGE, 6), State.DOWN, "Connection error: Closed at other end"); @@ -516,14 +518,14 @@ public class StateChangeTest extends FleetControllerTest { @Test void testDownNodeInitializing() throws Exception { // Actually report initializing state if node has been down steadily for a while - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 5000); - options.maxInitProgressTime = 5000; - options.stableStateTimePeriod = 20000; - options.nodeStateRequestTimeoutMS = 1000000; - options.maxSlobrokDisconnectGracePeriod = 1000000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxTransitionTime(NodeType.STORAGE, 5000) + .setMaxInitProgressTime(5000) + .setStableStateTimePeriod(20000) + .setNodeStateRequestTimeoutMS(1000000) + .setMaxSlobrokDisconnectGracePeriod(1000000); - initialize(options); + initialize(builder.build()); timer.advanceTime(100000); // Node has been in steady state up ctrl.tick(); @@ -579,13 +581,13 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNodeInitializationStalled() throws Exception { // Node should eventually be marked down, and not become initializing next time, but stay down until up - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 5000); - options.maxInitProgressTime = 5000; - options.stableStateTimePeriod = 1000000; - options.maxSlobrokDisconnectGracePeriod = 10000000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxTransitionTime(NodeType.STORAGE, 5000) + .setMaxInitProgressTime(5000) + .setStableStateTimePeriod(1000000) + .setMaxSlobrokDisconnectGracePeriod(10000000); - initialize(options); + initialize(builder.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -609,7 +611,7 @@ public class StateChangeTest extends FleetControllerTest { assertEquals("version:6 distributor:10 storage:10 .6.s:i .6.i:0.1", ctrl.getSystemState().toString()); - timer.advanceTime(options.maxInitProgressTime + 1); + timer.advanceTime(builder.maxInitProgressTime() + 1); ctrl.tick(); @@ -622,7 +624,7 @@ public class StateChangeTest extends FleetControllerTest { ctrl.tick(); - tick(options.nodeStateRequestTimeoutMS + 1); + tick(builder.nodeStateRequestTimeoutMS() + 1); communicator.setNodeState(new Node(NodeType.STORAGE, 6), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.0f), ""); @@ -665,14 +667,14 @@ public class StateChangeTest extends FleetControllerTest { @Test void testBackwardsInitializationProgress() throws Exception { // Same as stalled. Mark down, keep down until up - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 5000); - options.maxInitProgressTime = 5000; - options.stableStateTimePeriod = 1000000; - // Set long so we dont time out RPC requests and mark nodes down due to advancing time to get in steady state - options.nodeStateRequestTimeoutMS = (int) options.stableStateTimePeriod * 2; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)); + builder.setMaxTransitionTime(NodeType.STORAGE, 5000); + builder.setMaxInitProgressTime(5000); + builder.setStableStateTimePeriod(1000000); + // Set long so we don't time out RPC requests and mark nodes down due to advancing time to get in steady state + builder.setNodeStateRequestTimeoutMS((int) builder.stableStateTimePeriod() * 2); - initialize(options); + initialize(builder.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -708,13 +710,14 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNodeGoingDownWhileInitializing() throws Exception { // Same as stalled. Mark down, keep down until up - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 5000); - options.maxInitProgressTime = 5000; - options.stableStateTimePeriod = 1000000; - options.nodeStateRequestTimeoutMS = 365 * 24 * 60 * 1000; // Set very high so the advanceTime don't start sending state replies right before we disconnect. + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxTransitionTime(NodeType.STORAGE, 5000) + .setMaxInitProgressTime(5000) + .setStableStateTimePeriod(1000000) + // Set very high so the advanceTime don't start sending state replies right before we disconnect. + .setNodeStateRequestTimeoutMS(365 * 24 * 60 * 1000); - initialize(options); + initialize(builder.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -767,14 +770,14 @@ public class StateChangeTest extends FleetControllerTest { void testContinuousCrashRightAfterInit() throws Exception { startingTest("StateChangeTest::testContinuousCrashRightAfterInit"); // If node does this too many times, take it out of service - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 5000); - options.maxInitProgressTime = 5000; - options.maxPrematureCrashes = 2; - options.stableStateTimePeriod = 1000000; - options.maxSlobrokDisconnectGracePeriod = 10000000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxTransitionTime(NodeType.STORAGE, 5000) + .setMaxInitProgressTime(5000) + .setMaxPrematureCrashes(2) + .setStableStateTimePeriod(1000000) + .setMaxSlobrokDisconnectGracePeriod(10000000); - initialize(options); + initialize(builder.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -792,22 +795,22 @@ public class StateChangeTest extends FleetControllerTest { assertEquals("version:5 distributor:10 storage:10 .6.s:d", ctrl.getSystemState().toString()); - for (int j = 0; j <= options.maxPrematureCrashes; ++j) { + for (int j = 0; j <= builder.maxPrematureCrashes(); ++j) { ctrl.tick(); - tick(options.nodeStateRequestTimeoutMS + 1); + tick(builder.nodeStateRequestTimeoutMS() + 1); communicator.setNodeState(new Node(NodeType.STORAGE, 6), State.DOWN, "Connection error: Closed at other end"); ctrl.tick(); - tick(options.nodeStateRequestTimeoutMS + 1); + tick(builder.nodeStateRequestTimeoutMS() + 1); communicator.setNodeState(new Node(NodeType.STORAGE, 6), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.0f), ""); ctrl.tick(); - tick(options.nodeStateRequestTimeoutMS + 1); + tick(builder.nodeStateRequestTimeoutMS() + 1); communicator.setNodeState(new Node(NodeType.STORAGE, 6), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.1f), ""); @@ -821,15 +824,15 @@ public class StateChangeTest extends FleetControllerTest { void testClusterStateMinNodes() throws Exception { startingTest("StateChangeTest::testClusterStateMinNodes"); // If node does this too many times, take it out of service - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 0); - options.maxInitProgressTime = 0; - options.minDistributorNodesUp = 6; - options.minStorageNodesUp = 8; - options.minRatioOfDistributorNodesUp = 0.0; - options.minRatioOfStorageNodesUp = 0.0; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)) + .setMaxTransitionTime(NodeType.STORAGE, 0) + .setMaxInitProgressTime(0) + .setMinDistributorNodesUp(6) + .setMinStorageNodesUp(8) + .setMinRatioOfDistributorNodesUp(0.0) + .setMinRatioOfStorageNodesUp(0.0); - initialize(options); + initialize(builder.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -876,15 +879,15 @@ public class StateChangeTest extends FleetControllerTest { void testClusterStateMinFactor() throws Exception { startingTest("StateChangeTest::testClusterStateMinFactor"); // If node does this too many times, take it out of service - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 0); - options.maxInitProgressTime = 0; - options.minDistributorNodesUp = 0; - options.minStorageNodesUp = 0; - options.minRatioOfDistributorNodesUp = 0.6; - options.minRatioOfStorageNodesUp = 0.8; + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + options.setMaxTransitionTime(NodeType.STORAGE, 0); + options.setMaxInitProgressTime(0); + options.setMinDistributorNodesUp(0); + options.setMinStorageNodesUp(0); + options.setMinRatioOfDistributorNodesUp(0.6); + options.setMinRatioOfStorageNodesUp(0.8); - initialize(options); + initialize(options.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -949,9 +952,9 @@ public class StateChangeTest extends FleetControllerTest { @Test void testNoSystemStateBeforeInitialTimePeriod() throws Exception { startingTest("StateChangeTest::testNoSystemStateBeforeInitialTimePeriod()"); - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.minTimeBeforeFirstSystemStateBroadcast = 3 * 60 * 1000; - setUpSystem(options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)); + builder.setMinTimeBeforeFirstSystemStateBroadcast(3 * 60 * 1000); + setUpSystem(builder); boolean useFakeTimer = true; setUpVdsNodes(useFakeTimer, new DummyVdsNodeOptions(), true); // Leave one node down to avoid sending cluster state due to having seen all node states. @@ -960,7 +963,7 @@ public class StateChangeTest extends FleetControllerTest { nodes.get(i).connect(); } } - setUpFleetController(useFakeTimer, options); + setUpFleetController(useFakeTimer, builder); StateWaiter waiter = new StateWaiter(timer); fleetController.addSystemStateListener(waiter); @@ -995,11 +998,11 @@ public class StateChangeTest extends FleetControllerTest { @Test void testSystemStateSentWhenNodesReplied() throws Exception { startingTest("StateChangeTest::testSystemStateSentWhenNodesReplied()"); - final FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.minTimeBeforeFirstSystemStateBroadcast = 300 * 60 * 1000; + FleetControllerOptions.Builder builder = defaultOptions("mycluster", createNodes(10)); + builder.setMinTimeBeforeFirstSystemStateBroadcast(300 * 60 * 1000); boolean useFakeTimer = true; - setUpSystem(options); + setUpSystem(builder); setUpVdsNodes(useFakeTimer, new DummyVdsNodeOptions(), true); @@ -1009,7 +1012,7 @@ public class StateChangeTest extends FleetControllerTest { // Marking one node as 'initializing' improves testing of state later on. nodes.get(3).setNodeState(State.INITIALIZING); - setUpFleetController(useFakeTimer, options); + setUpFleetController(useFakeTimer, builder); final StateWaiter waiter = new StateWaiter(timer); @@ -1033,9 +1036,9 @@ public class StateChangeTest extends FleetControllerTest { @Test void testDontTagFailingSetSystemStateOk() throws Exception { startingTest("StateChangeTest::testDontTagFailingSetSystemStateOk()"); - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); StateWaiter waiter = new StateWaiter(timer); @@ -1064,10 +1067,10 @@ public class StateChangeTest extends FleetControllerTest { @Test void testAlteringDistributionSplitCount() throws Exception { startingTest("StateChangeTest::testAlteringDistributionSplitCount"); - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.distributionBits = 17; + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + options.setDistributionBits(17); - initialize(options); + initialize(options.build()); timer.advanceTime(1000000); // Node has been in steady state up @@ -1111,9 +1114,9 @@ public class StateChangeTest extends FleetControllerTest { @Test void testSetAllTimestampsAfterDowntime() throws Exception { startingTest("StateChangeTest::testSetAllTimestampsAfterDowntime"); - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); StateWaiter waiter = new StateWaiter(timer); @@ -1139,19 +1142,19 @@ public class StateChangeTest extends FleetControllerTest { } if (node.getNode().equals(new Node(NodeType.DISTRIBUTOR, 0))) { - for (ConfiguredNode i : options.nodes) { + for (ConfiguredNode i : options.nodes()) { Node nodeId = new Node(NodeType.STORAGE, i.index()); long ts = lastState.getNodeState(nodeId).getStartTimestamp(); assertTrue(ts > 0, nodeId + "\n" + stateHistory + "\nWas " + ts + " should be " + fleetController.getCluster().getNodeInfo(nodeId).getStartTimestamp()); } } else { - for (ConfiguredNode i : options.nodes) { + for (ConfiguredNode i : options.nodes()) { Node nodeId = new Node(NodeType.STORAGE, i.index()); assertEquals(0, lastState.getNodeState(nodeId).getStartTimestamp(), nodeId.toString()); } } - for (ConfiguredNode i : options.nodes) { + for (ConfiguredNode i : options.nodes()) { Node nodeId = new Node(NodeType.DISTRIBUTOR, i.index()); assertEquals(0, lastState.getNodeState(nodeId).getStartTimestamp(), nodeId.toString()); } @@ -1160,11 +1163,11 @@ public class StateChangeTest extends FleetControllerTest { @Test void consolidated_cluster_state_reflects_node_changes_when_cluster_is_down() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 0); - options.minStorageNodesUp = 10; - options.minDistributorNodesUp = 10; - initialize(options); + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + options.setMaxTransitionTime(NodeType.STORAGE, 0); + options.setMinStorageNodesUp(10); + options.setMinDistributorNodesUp(10); + initialize(options.build()); ctrl.tick(); assertThat(ctrl.consolidatedClusterState().toString(), equalTo("version:3 distributor:10 storage:10")); @@ -1194,11 +1197,11 @@ public class StateChangeTest extends FleetControllerTest { // of previous timer invocations (with subsequent state generation) would not be visible. @Test void timer_events_during_cluster_down_observe_most_recent_node_changes() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 1000); - options.minStorageNodesUp = 10; - options.minDistributorNodesUp = 10; - initialize(options); + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + options.setMaxTransitionTime(NodeType.STORAGE, 1000); + options.setMinStorageNodesUp(10); + options.setMinDistributorNodesUp(10); + initialize(options.build()); ctrl.tick(); communicator.setNodeState(new Node(NodeType.STORAGE, 2), State.DOWN, "foo"); @@ -1229,8 +1232,8 @@ public class StateChangeTest extends FleetControllerTest { @Test void do_not_emit_multiple_events_when_node_state_does_not_match_versioned_state() throws Exception { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - initialize(options); + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + initialize(options.build()); ctrl.tick(); communicator.setNodeState( @@ -1363,7 +1366,7 @@ public class StateChangeTest extends FleetControllerTest { void sendAllDeferredDistributorClusterStateAcks() throws Exception { communicator.sendAllDeferredDistributorClusterStateAcks(); ctrl.tick(); // Process cluster state bundle ACKs - if (ctrl.getOptions().enableTwoPhaseClusterStateActivation) { + if (ctrl.getOptions().enableTwoPhaseClusterStateActivation()) { ctrl.tick(); // Send activations ctrl.tick(); // Process activation ACKs } @@ -1371,7 +1374,7 @@ public class StateChangeTest extends FleetControllerTest { void processScheduledTask() throws Exception { ctrl.tick(); // Cluster state recompute iteration and send - if (ctrl.getOptions().enableTwoPhaseClusterStateActivation) { + if (ctrl.getOptions().enableTwoPhaseClusterStateActivation()) { ctrl.tick(); // Send activations ctrl.tick(); // Process activation ACKs } @@ -1399,20 +1402,20 @@ public class StateChangeTest extends FleetControllerTest { } } - private static FleetControllerOptions defaultOptions() { + private static FleetControllerOptions.Builder defaultOptions() { return defaultOptions("mycluster", createNodes(10)); } - private static FleetControllerOptions optionsWithZeroTransitionTime() { - FleetControllerOptions options = defaultOptions("mycluster", createNodes(10)); - options.maxTransitionTime.put(NodeType.STORAGE, 0); + private static FleetControllerOptions.Builder optionsWithZeroTransitionTime() { + FleetControllerOptions.Builder options = defaultOptions("mycluster", createNodes(10)); + options.setMaxTransitionTime(NodeType.STORAGE, 0); return options; } - private static FleetControllerOptions optionsAllowingZeroNodesDown() { - FleetControllerOptions options = optionsWithZeroTransitionTime(); - options.minStorageNodesUp = 10; - options.minDistributorNodesUp = 10; + private static FleetControllerOptions.Builder optionsAllowingZeroNodesDown() { + FleetControllerOptions.Builder options = optionsWithZeroTransitionTime(); + options.setMinStorageNodesUp(10); + options.setMinDistributorNodesUp(10); return options; } @@ -1421,7 +1424,7 @@ public class StateChangeTest extends FleetControllerTest { } private RemoteTaskFixture createDefaultFixture() throws Exception { - return new RemoteTaskFixture(defaultOptions()); + return new RemoteTaskFixture(defaultOptions().build()); } @Test @@ -1454,7 +1457,7 @@ public class StateChangeTest extends FleetControllerTest { @Test void no_op_synchronous_remote_task_can_complete_immediately_if_current_state_already_acked() throws Exception { - RemoteTaskFixture fixture = createFixtureWith(optionsWithZeroTransitionTime()); + RemoteTaskFixture fixture = createFixtureWith(optionsWithZeroTransitionTime().build()); fixture.markStorageNodeDown(0); MockTask task = fixture.scheduleNoOpVersionDependentTask(); // Tries to set node 0 into Down; already in that state @@ -1467,7 +1470,7 @@ public class StateChangeTest extends FleetControllerTest { @Test void no_op_synchronous_remote_task_waits_until_current_state_is_acked() throws Exception { - RemoteTaskFixture fixture = createFixtureWith(optionsWithZeroTransitionTime()); + RemoteTaskFixture fixture = createFixtureWith(optionsWithZeroTransitionTime().build()); communicator.setShouldDeferDistributorClusterStateAcks(true); fixture.markStorageNodeDown(0); @@ -1491,7 +1494,7 @@ public class StateChangeTest extends FleetControllerTest { // the cluster down-state to have been published. @Test void immediately_complete_sync_remote_task_when_cluster_is_down() throws Exception { - RemoteTaskFixture fixture = createFixtureWith(optionsAllowingZeroNodesDown()); + RemoteTaskFixture fixture = createFixtureWith(optionsAllowingZeroNodesDown().build()); // Controller options require 10/10 nodes up, so take one down to trigger a cluster Down edge. fixture.markStorageNodeDown(1); MockTask task = fixture.scheduleVersionDependentTaskWithSideEffects(); @@ -1523,12 +1526,12 @@ public class StateChangeTest extends FleetControllerTest { @Test void synchronous_task_immediately_failed_when_leadership_lost() throws Exception { - FleetControllerOptions options = optionsWithZeroTransitionTime(); - options.fleetControllerCount = 3; - RemoteTaskFixture fixture = createFixtureWith(options); + FleetControllerOptions.Builder options = optionsWithZeroTransitionTime(); + options.setCount(3); + RemoteTaskFixture fixture = createFixtureWith(options.build()); fixture.winLeadership(); - markAllNodesAsUp(options); + markAllNodesAsUp(options.build()); MockTask task = fixture.scheduleVersionDependentTaskWithSideEffects(); @@ -1548,9 +1551,9 @@ public class StateChangeTest extends FleetControllerTest { @Test void cluster_state_ack_is_not_dependent_on_state_send_grace_period() throws Exception { - FleetControllerOptions options = defaultOptions(); - options.minTimeBetweenNewSystemStates = 10_000; - RemoteTaskFixture fixture = createFixtureWith(options); + FleetControllerOptions.Builder options = defaultOptions(); + options.setMinTimeBetweenNewSystemStates(10_000); + RemoteTaskFixture fixture = createFixtureWith(options.build()); // Have to increment timer here to be able to send state generated by the scheduled task timer.advanceTime(10_000); @@ -1568,8 +1571,9 @@ public class StateChangeTest extends FleetControllerTest { @Test void synchronous_task_immediately_answered_when_not_leader() throws Exception { - FleetControllerOptions options = optionsWithZeroTransitionTime(); - options.fleetControllerCount = 3; + FleetControllerOptions.Builder builder = optionsWithZeroTransitionTime(); + builder.setCount(3); + var options = builder.build(); RemoteTaskFixture fixture = createFixtureWith(options); fixture.loseLeadership(); @@ -1583,9 +1587,9 @@ public class StateChangeTest extends FleetControllerTest { @Test void task_not_completed_within_deadline_is_failed_with_deadline_exceeded_error() throws Exception { - FleetControllerOptions options = defaultOptions(); - options.setMaxDeferredTaskVersionWaitTime(Duration.ofSeconds(60)); - RemoteTaskFixture fixture = createFixtureWith(options); + FleetControllerOptions.Builder builder = defaultOptions(); + builder.setMaxDeferredTaskVersionWaitTime(Duration.ofSeconds(60)); + RemoteTaskFixture fixture = createFixtureWith(builder.build()); MockTask task = fixture.scheduleVersionDependentTaskWithSideEffects(); communicator.setShouldDeferDistributorClusterStateAcks(true); @@ -1607,11 +1611,11 @@ public class StateChangeTest extends FleetControllerTest { } private void doTestTaskDeadlineExceeded(boolean deferredActivation, String expectedMessage) throws Exception { - FleetControllerOptions options = defaultOptions(); + FleetControllerOptions.Builder options = defaultOptions(); options.setMaxDeferredTaskVersionWaitTime(Duration.ofSeconds(60)); - options.enableTwoPhaseClusterStateActivation = deferredActivation; - options.maxDivergentNodesPrintedInTaskErrorMessages = 10; - RemoteTaskFixture fixture = createFixtureWith(options); + options.enableTwoPhaseClusterStateActivation(deferredActivation); + options.setMaxDivergentNodesPrintedInTaskErrorMessages(10); + RemoteTaskFixture fixture = createFixtureWith(options.build()); MockTask task = fixture.scheduleVersionDependentTaskWithSideEffects(); communicator.setShouldDeferDistributorClusterStateAcks(true); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java index 656d5a187d1..6ecfc9ca550 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.NodeType; import org.junit.jupiter.api.Test; import java.time.Instant; import java.util.concurrent.TimeoutException; @@ -27,16 +28,16 @@ public class StateGatherTest extends FleetControllerTest { void testAlwaysHavePendingGetNodeStateRequestTowardsNodes() throws Exception { Logger.getLogger(NodeStateGatherer.class.getName()).setLevel(Level.FINEST); startingTest("StateGatherTest::testOverlappingGetNodeStateRequests"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.nodeStateRequestTimeoutMS = 10 * 60 * 1000; - // Force actual message timeout to be lower than request timeout. - options.nodeStateRequestTimeoutEarliestPercentage = 80; - options.nodeStateRequestTimeoutLatestPercentage = 80; - setUpFleetController(true, options); + FleetControllerOptions.Builder builder = defaultOptions("mycluster") + .setNodeStateRequestTimeoutMS(10 * 60 * 1000) + // Force actual message timeout to be lower than request timeout. + .setNodeStateRequestTimeoutEarliestPercentage(80) + .setNodeStateRequestTimeoutLatestPercentage(80); + setUpFleetController(true, builder); String[] connectionSpecs = getSlobrokConnectionSpecs(slobrok); DummyVdsNodeOptions dummyOptions = new DummyVdsNodeOptions(); - DummyVdsNode dnode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, this.options.clusterName, true, 0); - DummyVdsNode snode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, this.options.clusterName, false, 0); + DummyVdsNode dnode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, builder.clusterName(), NodeType.DISTRIBUTOR, 0); + DummyVdsNode snode = new DummyVdsNode(timer, dummyOptions, connectionSpecs, builder.clusterName(), NodeType.STORAGE, 0); dnode.connect(); snode.connect(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java index 7bbae78dbab..e89f5fc7e58 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java @@ -12,7 +12,7 @@ public class WantedStateTest extends FleetControllerTest { void testSettingStorageNodeMaintenanceAndBack() throws Exception { startingTest("WantedStateTest::testSettingStorageNodeMaintenanceAndBack()"); setUpFleetController(true, defaultOptions("mycluster")); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); setWantedState(nodes.get(1), State.MAINTENANCE, null); @@ -26,7 +26,7 @@ public class WantedStateTest extends FleetControllerTest { void testOverridingWantedStateOtherReason() throws Exception { startingTest("WantedStateTest::testOverridingWantedStateOtherReason()"); setUpFleetController(true, defaultOptions("mycluster")); - setUpVdsNodes(true, new DummyVdsNodeOptions()); + setUpVdsNodes(true); waitForStableSystem(); setWantedState(nodes.get(1), State.MAINTENANCE, "Foo"); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicatorTest.java index b9a9d7fbf8f..7e0b7f6d953 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicatorTest.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.clustercontroller.core.SetClusterStateRequest; import com.yahoo.vespa.clustercontroller.core.Timer; import org.junit.jupiter.api.Test; import org.mockito.Mockito; - import java.time.Duration; import java.util.HashSet; import java.util.Set; @@ -34,7 +33,6 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -79,11 +77,11 @@ public class RPCCommunicatorTest { @Test void testGenerateNodeStateRequestTimeoutMsWithUpdates() { final RPCCommunicator communicator = new RPCCommunicator(RPCCommunicator.createRealSupervisor(), null /* Timer */, INDEX, 1, 1, 100, 0); - FleetControllerOptions fleetControllerOptions = new FleetControllerOptions(null /*clustername*/, Set.of(new ConfiguredNode(0, false))); - fleetControllerOptions.nodeStateRequestTimeoutEarliestPercentage = 100; - fleetControllerOptions.nodeStateRequestTimeoutLatestPercentage = 100; - fleetControllerOptions.nodeStateRequestTimeoutMS = NODE_STATE_REQUEST_TIMEOUT_INTERVAL_MAX_MS; - communicator.propagateOptions(fleetControllerOptions); + FleetControllerOptions.Builder builder = new FleetControllerOptions.Builder(null /*clustername*/, Set.of(new ConfiguredNode(0, false))); + builder.setNodeStateRequestTimeoutEarliestPercentage(100); + builder.setNodeStateRequestTimeoutLatestPercentage(100); + builder.setNodeStateRequestTimeoutMS(NODE_STATE_REQUEST_TIMEOUT_INTERVAL_MAX_MS); + communicator.propagateOptions(builder.build()); long timeOutMs = communicator.generateNodeStateRequestTimeout().toMillis(); assertEquals(timeOutMs, NODE_STATE_REQUEST_TIMEOUT_INTERVAL_MAX_MS); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java index 7960cd7c9d2..e4df6f31987 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java @@ -117,6 +117,7 @@ public interface Waiter { if (allowWait) data.getMonitor().wait(wt == null ? WaitTask.defaultTaskFrequencyMillis : Math.min(wt.getWaitTaskFrequencyInMillis(), timeLeft.toMillis())); } catch (InterruptedException e) { + throw new RuntimeException(e); } } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index f439e88f260..7973ed93db2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -73,10 +73,11 @@ public interface ModelContext { */ interface FeatureFlags { @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { return "THROUGHPUT"; } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String queryDispatchPolicy() { return "adaptive"; } + @ModelFeatureFlag(owners = {"baldersheim"}) default String phraseOptimization() { return ""; } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipCommunicationManagerThread() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusRequestThread() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index c2218cef684..57f01fd8f55 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,6 +42,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; private String queryDispatchPolicy = "adaptive"; + private String phraseOptimization = ""; private String sequencerType = "THROUGHPUT"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; @@ -151,6 +152,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } + @Override public String phraseOptimization() { return phraseOptimization; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { @@ -200,6 +202,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea queryDispatchPolicy = policy; return this; } + public TestProperties setPhraseOptimization(String value) { + phraseOptimization = value; + return this; + } public TestProperties setFeedSequencerType(String type) { sequencerType = type; return this; diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 56786c733ec..dc7605c4897 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.google.common.collect.ImmutableMap; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; @@ -27,7 +28,6 @@ import com.yahoo.tensor.TensorType; import java.io.IOException; import java.io.Reader; -import java.io.Serializable; import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; @@ -118,7 +118,7 @@ public class RankProfile implements Cloneable { private Map<Reference, Constant> constants = new LinkedHashMap<>(); - private Map<String, OnnxModel> onnxModels = new LinkedHashMap<>(); + private final Map<String, OnnxModel> onnxModels = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -142,11 +142,9 @@ public class RankProfile implements Cloneable { * and looking up rank profiles. */ public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { - this.name = Objects.requireNonNull(name, "name cannot be null"); - this.schema = Objects.requireNonNull(schema, "schema cannot be null"); - this.rankProfileRegistry = rankProfileRegistry; - this.applicationPackage = schema.applicationPackage(); - this.deployLogger = schema.getDeployLogger(); + this(name, Objects.requireNonNull(schema, "schema cannot be null"), + schema.applicationPackage(), schema.getDeployLogger(), + schema.getDeployProperties(), rankProfileRegistry); } /** @@ -154,13 +152,18 @@ public class RankProfile implements Cloneable { * * @param name the name of the new profile */ - public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry) { + public RankProfile(String name, Schema schema, ApplicationPackage applicationPackage, DeployLogger deployLogger, + ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.schema = null; + this.schema = schema; this.rankProfileRegistry = rankProfileRegistry; this.applicationPackage = applicationPackage; this.deployLogger = deployLogger; + if (deployProperties.featureFlags().phraseOptimization().contains("split")) { + addRankProperty(new RankProperty("vespa.matching.split_unpacking_iterators", "true")); + } else if (deployProperties.featureFlags().phraseOptimization().contains("off")) { + addRankProperty(new RankProperty("vespa.matching.split_unpacking_iterators", "false")); + } } public String name() { return name; } @@ -317,7 +320,7 @@ public class RankProfile implements Cloneable { .filter(p -> nonEmptyValueFilter.test(p)) .collect(Collectors.toSet()); if (uniqueProperties.isEmpty()) return Optional.empty(); - if (uniqueProperties.size() == 1) return Optional.of(uniqueProperties.stream().findAny().get()); + if (uniqueProperties.size() == 1) return uniqueProperties.stream().findAny(); throw new IllegalArgumentException("Only one of the profiles inherited by " + this + " can contain " + propertyDescription + ", but it is present in multiple"); } @@ -641,7 +644,7 @@ public class RankProfile implements Cloneable { /** Returns a read only map view of the rank properties to use in this profile. This is never null. */ public Map<String, List<RankProperty>> getRankPropertyMap() { - if (rankProperties.size() == 0 && inherited().isEmpty()) return Map.of(); + if (rankProperties.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") @@ -1148,7 +1151,7 @@ public class RankProfile implements Cloneable { * A rank setting. The identity of a rank setting is its field name and type (not value). * A rank setting is immutable. */ - public static class RankSetting implements Serializable { + public static class RankSetting { private final String fieldName; @@ -1222,13 +1225,10 @@ public class RankProfile implements Cloneable { @Override public boolean equals(Object object) { - if (!(object instanceof RankSetting)) { + if (!(object instanceof RankSetting other)) { return false; } - RankSetting other = (RankSetting)object; - return - fieldName.equals(other.fieldName) && - type.equals(other.type); + return fieldName.equals(other.fieldName) && type.equals(other.type); } @Override @@ -1239,7 +1239,7 @@ public class RankProfile implements Cloneable { } /** A rank property. Rank properties are Value Objects */ - public static class RankProperty implements Serializable { + public static class RankProperty { private final String name; private final String value; @@ -1260,8 +1260,7 @@ public class RankProfile implements Cloneable { @Override public boolean equals(Object object) { - if (! (object instanceof RankProperty)) return false; - RankProperty other=(RankProperty)object; + if (! (object instanceof RankProperty other)) return false; return (other.name.equals(this.name) && other.value.equals(this.value)); } @@ -1482,8 +1481,7 @@ public class RankProfile implements Cloneable { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof Constant)) return false; - Constant other = (Constant)o; + if ( ! (o instanceof Constant other)) return false; if ( ! other.name().equals(this.name())) return false; if ( ! other.type().equals(this.type())) return false; if ( ! other.value().equals(this.value())) return false; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java index b378d127430..505778ad047 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -32,8 +32,8 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce public enum FieldSet {ALL, FAST_ACCESS} - private Map<String, Attribute> attributes = new java.util.LinkedHashMap<>(); - private Map<String, Attribute> importedAttributes = new java.util.LinkedHashMap<>(); + private final Map<String, Attribute> attributes = new java.util.LinkedHashMap<>(); + private final Map<String, Attribute> importedAttributes = new java.util.LinkedHashMap<>(); /** Whether this has any position attribute */ private boolean hasPosition = false; @@ -156,7 +156,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } /** Returns a read only attribute iterator */ - public Iterator attributeIterator() { + public Iterator<Attribute> attributeIterator() { return attributes().iterator(); } @@ -257,33 +257,24 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } private static AttributesConfig.Attribute.Dictionary.Type.Enum convert(Dictionary.Type type) { - switch (type) { - case BTREE: - return AttributesConfig.Attribute.Dictionary.Type.BTREE; - case HASH: - return AttributesConfig.Attribute.Dictionary.Type.HASH; - case BTREE_AND_HASH: - return AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; - } - return AttributesConfig.Attribute.Dictionary.Type.BTREE; + return switch (type) { + case BTREE: yield AttributesConfig.Attribute.Dictionary.Type.BTREE; + case HASH: yield AttributesConfig.Attribute.Dictionary.Type.HASH; + case BTREE_AND_HASH: yield AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; + + }; } private static AttributesConfig.Attribute.Dictionary.Match.Enum convert(Case type) { - switch (type) { - case CASED: - return AttributesConfig.Attribute.Dictionary.Match.CASED; - case UNCASED: - return AttributesConfig.Attribute.Dictionary.Match.UNCASED; - } - return AttributesConfig.Attribute.Dictionary.Match.UNCASED; + return switch (type) { + case CASED: yield AttributesConfig.Attribute.Dictionary.Match.CASED; + case UNCASED: yield AttributesConfig.Attribute.Dictionary.Match.UNCASED; + }; } private static AttributesConfig.Attribute.Match.Enum convertMatch(Case type) { - switch (type) { - case CASED: - return AttributesConfig.Attribute.Match.CASED; - case UNCASED: - return AttributesConfig.Attribute.Match.UNCASED; - } - return AttributesConfig.Attribute.Match.UNCASED; + return switch (type) { + case CASED: yield AttributesConfig.Attribute.Match.CASED; + case UNCASED: yield AttributesConfig.Attribute.Match.UNCASED; + }; } public void getConfig(AttributesConfig.Builder builder, FieldSet fs, long maxUnCommittedMemory, boolean enableBitVectors) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index efc64a5aa40..9cd5d901574 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -59,7 +59,7 @@ public class SummaryClass extends Derived { /** MUST be called after all other fields are added */ private void deriveImplicitFields(DocumentSummary summary, Map<String, SummaryClassField> fields) { if (summary.getName().equals("default")) { - addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, SummaryTransform.DOCUMENT_ID, fields); + addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, SummaryTransform.DOCUMENT_ID, "", fields); } } @@ -68,12 +68,14 @@ public class SummaryClass extends Derived { if (!accessingDiskSummary && schema.isAccessingDiskSummary(summaryField)) { accessingDiskSummary = true; } - addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), fields); + addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), + SummaryMap.getSource(summaryField), fields); } } private void addField(String name, DataType type, SummaryTransform transform, + String source, Map<String, SummaryClassField> fields) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); @@ -82,7 +84,7 @@ public class SummaryClass extends Derived { ". " + "Declared as type " + sf.getType() + " and " + type); } } else { - fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); + fields.put(name, new SummaryClassField(name, type, transform, source, rawAsBase64)); } } @@ -110,7 +112,9 @@ public class SummaryClass extends Derived { for (SummaryClassField field : fields.values() ) { classBuilder.fields(new SummaryConfig.Classes.Fields.Builder(). name(field.getName()). - type(field.getType().getName())); + type(field.getType().getName()). + command(field.getCommand()). + source(field.getSource())); } return classBuilder; } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index f042054a0b5..00972895306 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -28,8 +28,9 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; public class SummaryClassField { private final String name; - private final Type type; + private final String command; + private final String source; /** The summary field type enumeration */ public enum Type { @@ -68,15 +69,21 @@ public class SummaryClassField { } } - public SummaryClassField(String name, DataType type, SummaryTransform transform, boolean rawAsBase64) { + public SummaryClassField(String name, DataType type, SummaryTransform transform, String source, boolean rawAsBase64) { this.name = name; this.type = convertDataType(type, transform, rawAsBase64); + this.command = SummaryMap.getCommand(transform); + this.source = source; } public String getName() { return name; } public Type getType() { return type; } + public String getCommand() { return command; } + + public String getSource() { return source; } + /** Converts to the right summary field type from a field datatype and a transform*/ public static Type convertDataType(DataType fieldType, SummaryTransform transform, boolean rawAsBase64) { FieldValue fval = fieldType.createFieldValue(); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java index 9d3d00f1481..cecf3ecdd85 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java @@ -49,31 +49,14 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { private void derive(DocumentSummary documentSummary) { for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { - if (summaryField.getTransform()== SummaryTransform.NONE) continue; - - if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || - (summaryField.getTransform()==SummaryTransform.ATTRIBUTECOMBINER && summaryField.hasExplicitSingleSource()) || - summaryField.getTransform()==SummaryTransform.COPY || - summaryField.getTransform()==SummaryTransform.DISTANCE || - summaryField.getTransform()==SummaryTransform.GEOPOS || - summaryField.getTransform()==SummaryTransform.POSITIONS || - summaryField.getTransform()==SummaryTransform.MATCHED_ELEMENTS_FILTER || - summaryField.getTransform()==SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) - { - resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getSingleSource())); - } else { - // Note: Currently source mapping is handled in the indexing statement, - // by creating a summary field for each of the values - // This works, but is suboptimal. We could consolidate to a minimal set and - // use the right value from the minimal set as the third parameter here, - // and add "override" commands to multiple static values - boolean useFieldNameAsArgument = summaryField.getTransform().isDynamic(); - resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - useFieldNameAsArgument ? summaryField.getName() : "")); + if (summaryField.getTransform()== SummaryTransform.NONE) { + continue; } + + resultTransforms.put(summaryField.getName(), + new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + getSource(summaryField))); } } @@ -85,13 +68,42 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { protected String getDerivedName() { return "summarymap"; } /** Returns the command name of a transform */ - private String getCommand(SummaryTransform transform) { - if (transform == SummaryTransform.DISTANCE) + static String getCommand(SummaryTransform transform) { + if (transform == SummaryTransform.NONE) { + return ""; + } else if (transform == SummaryTransform.DISTANCE) { return "absdist"; - else if (transform.isDynamic()) + } else if (transform.isDynamic()) { return "dynamicteaser"; - else + } else { return transform.getName(); + } + } + + static String getSource(SummaryField summaryField) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + return ""; + } + + if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE || + (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER && summaryField.hasExplicitSingleSource()) || + summaryField.getTransform() == SummaryTransform.COPY || + summaryField.getTransform() == SummaryTransform.DISTANCE || + summaryField.getTransform() == SummaryTransform.GEOPOS || + summaryField.getTransform() == SummaryTransform.POSITIONS || + summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || + summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) + { + return summaryField.getSingleSource(); + } else { + // Note: Currently source mapping is handled in the indexing statement, + // by creating a summary field for each of the values + // This works, but is suboptimal. We could consolidate to a minimal set and + // use the right value from the minimal set as the third parameter here, + // and add "override" commands to multiple static values + boolean useFieldNameAsArgument = summaryField.getTransform().isDynamic(); + return useFieldNameAsArgument ? summaryField.getName() : ""; + } } /** @@ -100,7 +112,7 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { * A dynamic transform needs the query to perform its computations. */ // TODO/Note: "dynamic" here means something else than in SummaryTransform - public static boolean isDynamicCommand(String commandName) { + static boolean isDynamicCommand(String commandName) { return (commandName.equals("dynamicteaser") || commandName.equals(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()) || commandName.equals(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 7f5a78fa917..9bb03d3f07c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -17,10 +17,7 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.api.ApplicationClusterInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.Model; -import com.yahoo.config.model.api.Provisioned; +import com.yahoo.config.model.api.*; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -61,7 +58,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import org.xml.sax.SAXException; import java.io.IOException; -import java.io.Serializable; import java.lang.reflect.Constructor; import java.time.Instant; import java.util.ArrayList; @@ -96,9 +92,7 @@ import static java.util.stream.Collectors.toUnmodifiableMap; * * @author gjoranv */ -public final class VespaModel extends AbstractConfigProducerRoot implements Serializable, Model { - - private static final long serialVersionUID = 1L; +public final class VespaModel extends AbstractConfigProducerRoot implements Model { public static final Logger log = Logger.getLogger(VespaModel.class.getName()); @@ -264,11 +258,13 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri DeployLogger deployLogger = deployState.getDeployLogger(); RankProfileRegistry rankProfileRegistry = deployState.rankProfileRegistry(); QueryProfiles queryProfiles = deployState.getQueryProfiles(); + ModelContext.Properties deployProperties = deployState.getProperties(); List <Future<ConvertedModel>> futureModels = new ArrayList<>(); if ( ! importedModels.isEmpty()) { // models/ directory is available for (ImportedMlModel model : importedModels) { // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. - RankProfile profile = new RankProfile(model.name(), applicationPackage, deployLogger, rankProfileRegistry); + RankProfile profile = new RankProfile(model.name(), null, applicationPackage, + deployLogger, deployProperties, rankProfileRegistry); addOnnxModelInfoFromSource(model, profile); rankProfileRegistry.add(profile); futureModels.add(deployState.getExecutor().submit(() -> { @@ -285,7 +281,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri String modelName = generatedModelDir.getPath().last(); if (modelName.contains(".")) continue; // Name space: Not a global profile // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. - RankProfile profile = new RankProfile(modelName, applicationPackage, deployLogger, rankProfileRegistry); + RankProfile profile = new RankProfile(modelName, null, applicationPackage, + deployLogger, deployProperties, rankProfileRegistry); addOnnxModelInfoFromStore(modelName, profile); rankProfileRegistry.add(profile); futureModels.add(deployState.getExecutor().submit(() -> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index bbccdaa9453..98b8574a015 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -81,6 +81,9 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } + if (options.hostedVespa().orElse(false)) { + builder.vespaTlsConfigFile(Defaults.getDefaults().underVespaHome("conf/zookeeper/tls.conf.json")); + } } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index acd8b5cbbc2..0bf586a089f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -1037,7 +1037,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { */ private static class JvmOptions { - private static final Pattern validPattern = Pattern.compile("-[a-zA-z0-9=:./,+-]+"); + private static final Pattern validPattern = Pattern.compile("-[a-zA-z0-9=:./,+*-]+"); // debug port will not be available in hosted, don't allow private static final Pattern invalidInHostedatttern = Pattern.compile("-Xrunjdwp:transport=.*"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 74017b6b821..bf5770681ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -117,20 +117,23 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce RedundancyBuilder redundancyBuilder = new RedundancyBuilder(contentElement); Set<NewDocumentType> globallyDistributedDocuments = new GlobalDistributionBuilder(documentDefinitions).build(documentsElement); - ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, + String clusterId = getClusterId(contentElement); + ContentCluster c = new ContentCluster(context.getParentProducer(), clusterId, documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); var resourceLimits = new ClusterResourceLimits.Builder(stateIsHosted(deployState), deployState.featureFlags().resourceLimitDisk(), deployState.featureFlags().resourceLimitMemory()) .build(contentElement); - c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), - contentElement, - resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(clusterId, + contentElement, + resourceLimits.getClusterControllerLimits()) + .build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, - globallyDistributedDocuments, - fractionOfMemoryReserved(getClusterId(contentElement), containers), - resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml()); + globallyDistributedDocuments, + fractionOfMemoryReserved(clusterId, containers), + resourceLimits.getContentNodeLimits()) + .build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); @@ -173,9 +176,8 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce if (csc.hasIndexedCluster()) { setupIndexedCluster(csc.getIndexed(), search, element, logger); } - - } + private void setupIndexedCluster(IndexedSearchCluster index, ContentSearch search, ModelElement element, DeployLogger logger) { Double queryTimeout = search.getQueryTimeout(); if (queryTimeout != null) { @@ -267,9 +269,8 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } private void validateThatGroupSiblingsAreUnique(String cluster, StorageGroup group) { - if (group == null) { - return; // Unit testing case - } + if (group == null) return; // Unit testing case + validateGroupSiblings(cluster, group); for (StorageGroup g : group.getSubgroups()) { validateThatGroupSiblingsAreUnique(cluster, g); @@ -301,7 +302,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } clusterControllers = admin.getClusterControllers(); } - else { // self-hosted: Put cluser controller on config servers or use explicit cluster controllers + else { // self-hosted: Put cluster controller on config servers or use explicit cluster controllers if (admin.getClusterControllers() == null) { var hosts = admin.getConfigservers().stream().map(s -> s.getHostResource()).collect(toList()); if (hosts.size() > 1) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index fb4016f4cf4..ff905187969 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -44,11 +44,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final Integer numThreads; private final ContentCluster cluster; - private final int reponseNumThreads; + private final int responseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; - private final int persistenceThrottingWindowSize; + private final int persistenceThrottlingWindowSize; private final double persistenceThrottlingWsResizeRate; private final boolean persistenceThrottlingOfMergeFeedOps; private final boolean useAsyncMessageHandlingOnSchedule; @@ -64,11 +64,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; - this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); + this.responseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); - this.persistenceThrottingWindowSize = featureFlags.persistenceThrottlingWindowSize(); + this.persistenceThrottlingWindowSize = featureFlags.persistenceThrottlingWindowSize(); this.persistenceThrottlingWsResizeRate = featureFlags.persistenceThrottlingWsResizeRate(); this.persistenceThrottlingOfMergeFeedOps = featureFlags.persistenceThrottlingOfMergeFeedOps(); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); @@ -80,15 +80,15 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_threads(numThreads); } builder.enable_multibit_split_optimalization(cluster.getPersistence().enableMultiLevelSplitting()); - builder.num_response_threads(reponseNumThreads); + builder.num_response_threads(responseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); - if (persistenceThrottingWindowSize > 0) { - throttleBuilder.min_window_size(persistenceThrottingWindowSize); - throttleBuilder.max_window_size(persistenceThrottingWindowSize); + if (persistenceThrottlingWindowSize > 0) { + throttleBuilder.min_window_size(persistenceThrottlingWindowSize); + throttleBuilder.max_window_size(persistenceThrottlingWindowSize); } throttleBuilder.resize_rate(persistenceThrottlingWsResizeRate); throttleBuilder.throttle_individual_merge_feed_ops(persistenceThrottlingOfMergeFeedOps); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java index 588e7c55ab9..4f81bbf165f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java @@ -16,9 +16,9 @@ public class IntegrityCheckerProducer implements StorIntegritycheckerConfig.Prod } } - private Integer startTime; - private Integer stopTime; - private String weeklyCycle; + private final Integer startTime; + private final Integer stopTime; + private final String weeklyCycle; IntegrityCheckerProducer(Integer startTime, Integer stopTime, String weeklyCycle) { this.startTime = startTime; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 2fca964a995..e66f2c48f26 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -3,10 +3,8 @@ package com.yahoo.vespa.model.content.storagecluster; import com.yahoo.config.model.api.ModelContext; import com.yahoo.vespa.config.content.core.StorServerConfig; -import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; - -import java.util.Optional; +import com.yahoo.vespa.model.content.cluster.ContentCluster; /** * Serves config for stor-server for storage clusters (clusters of storage nodes). @@ -28,11 +26,10 @@ public class StorServerProducer implements StorServerConfig.Producer { } } - private String clusterName; + private final String clusterName; private Integer maxMergesPerNode; private Integer queueSize; - private Integer bucketDBStripeBits; - private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; + private final StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -46,10 +43,6 @@ public class StorServerProducer implements StorServerConfig.Producer { } return this; } - private StorServerProducer setBucketDBStripeBits(Integer value) { - bucketDBStripeBits = value; - return this; - } private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { try { @@ -80,9 +73,6 @@ public class StorServerProducer implements StorServerConfig.Producer { if (queueSize != null) { builder.max_merge_queue_size(queueSize); } - if (bucketDBStripeBits != null) { - builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); - } // TODO set throttle policy params based on existing or separate flags builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index a3f32fcb44b..da82a69842a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -30,7 +30,7 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> { public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageCluster> { @Override - protected StorageCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { + protected StorageCluster doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element producerSpec) { final ModelElement clusterElem = new ModelElement(producerSpec); final ContentCluster cluster = (ContentCluster)ancestor; @@ -51,7 +51,7 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> private final StorVisitorProducer storVisitorProducer; private final PersistenceProducer persistenceProducer; - StorageCluster(AbstractConfigProducer parent, + StorageCluster(AbstractConfigProducer<?> parent, String clusterName, FileStorProducer fileStorProducer, IntegrityCheckerProducer integrityCheckerProducer, diff --git a/config-model/src/test/derived/advanced/summary.cfg b/config-model/src/test/derived/advanced/summary.cfg index 18d67a9ea84..11a73d6a90c 100644 --- a/config-model/src/test/derived/advanced/summary.cfg +++ b/config-model/src/test/derived/advanced/summary.cfg @@ -5,28 +5,52 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "debug" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "attributes" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "title_s" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "product" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "product3" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "mysummary" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 472092010 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "location_zcurve" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "location_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg index 38298feaa0c..e62a8a5c39c 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -5,18 +5,32 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "elem_array" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 659145226 classes[].name "rename" classes[].omitsummaryfeatures false classes[].fields[].name "new_elem_array" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "elem_array" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg index 7fabb674ebd..e2ed135c3a2 100644 --- a/config-model/src/test/derived/attributeprefetch/summary.cfg +++ b/config-model/src/test/derived/attributeprefetch/summary.cfg @@ -5,26 +5,48 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1980470965 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "singlebyte" classes[].fields[].type "byte" +classes[].fields[].command "attribute" +classes[].fields[].source "singlebyte" classes[].fields[].name "singleint" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "singleint" classes[].fields[].name "singlelong" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "singlelong" classes[].fields[].name "singlefloat" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "singlefloat" classes[].fields[].name "singledouble" classes[].fields[].type "double" +classes[].fields[].command "attribute" +classes[].fields[].source "singledouble" classes[].fields[].name "singlestring" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "singlestring" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg index 44b97327ee9..1d6f7141635 100644 --- a/config-model/src/test/derived/complex/summary.cfg +++ b/config-model/src/test/derived/complex/summary.cfg @@ -5,40 +5,76 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "woe" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "exact" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "dyntitle" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "dyntitle" classes[].fields[].name "source" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "stringfield" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 128090024 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "year_sub" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year_sub" classes[].fields[].name "prefixenabled" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "prefixenabled" classes[].fields[].name "fleeting2" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "fleeting2" classes[].fields[].name "foundat" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "foundat" classes[].fields[].name "collapseby" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "collapseby" classes[].fields[].name "ts" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "ts" classes[].fields[].name "combineda" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "combineda" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/emptychild/summary.cfg b/config-model/src/test/derived/emptychild/summary.cfg index 116bad51740..5fe39cbb04c 100644 --- a/config-model/src/test/derived/emptychild/summary.cfg +++ b/config-model/src/test/derived/emptychild/summary.cfg @@ -5,18 +5,32 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "a1" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a1" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1490368133 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "a1" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a1" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/emptydefault/summary.cfg b/config-model/src/test/derived/emptydefault/summary.cfg index f1a858d2d1e..c09b606210f 100644 --- a/config-model/src/test/derived/emptydefault/summary.cfg +++ b/config-model/src/test/derived/emptydefault/summary.cfg @@ -5,7 +5,13 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/id/summary.cfg b/config-model/src/test/derived/id/summary.cfg index 6880a8a7bd5..203be8549b8 100644 --- a/config-model/src/test/derived/id/summary.cfg +++ b/config-model/src/test/derived/id/summary.cfg @@ -5,9 +5,17 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "uri" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg index b324b4cd8e4..c3f3e45e2f0 100644 --- a/config-model/src/test/derived/imported_position_field/summary.cfg +++ b/config-model/src/test/derived/imported_position_field/summary.cfg @@ -5,16 +5,28 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "parent_ref" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1274088866 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/imported_position_field_summary/summary.cfg b/config-model/src/test/derived/imported_position_field_summary/summary.cfg index cf80a3f31d0..06fca8c89a1 100644 --- a/config-model/src/test/derived/imported_position_field_summary/summary.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/summary.cfg @@ -5,27 +5,47 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "parent_ref" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "my_pos" classes[].fields[].type "jsonstring" +classes[].fields[].command "geopos" +classes[].fields[].source "my_pos_zcurve" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 656588065 classes[].name "mysummary" classes[].omitsummaryfeatures false classes[].fields[].name "my_pos" classes[].fields[].type "jsonstring" +classes[].fields[].command "geopos" +classes[].fields[].source "my_pos_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1274088866 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/imported_struct_fields/summary.cfg b/config-model/src/test/derived/imported_struct_fields/summary.cfg index 38fdd23123b..46a6a7db64f 100644 --- a/config-model/src/test/derived/imported_struct_fields/summary.cfg +++ b/config-model/src/test/derived/imported_struct_fields/summary.cfg @@ -5,44 +5,78 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "parent_ref" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 2126652894 classes[].name "mysummary" classes[].omitsummaryfeatures false classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].fields[].name "my_elem_array" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "" classes[].fields[].name "my_elem_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "" classes[].fields[].name "my_str_int_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1629947863 classes[].name "filtered" classes[].omitsummaryfeatures false classes[].fields[].name "elem_array_filtered" classes[].fields[].type "jsonstring" +classes[].fields[].command "matchedattributeelementsfilter" +classes[].fields[].source "my_elem_array" classes[].fields[].name "elem_map_filtered" classes[].fields[].type "jsonstring" +classes[].fields[].command "matchedattributeelementsfilter" +classes[].fields[].source "my_elem_map" classes[].fields[].name "str_int_map_filtered" classes[].fields[].type "jsonstring" +classes[].fields[].command "matchedattributeelementsfilter" +classes[].fields[].source "my_str_int_map" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1274088866 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/importedfields/summary.cfg b/config-model/src/test/derived/importedfields/summary.cfg index 4b792d15de9..2614bea2092 100644 --- a/config-model/src/test/derived/importedfields/summary.cfg +++ b/config-model/src/test/derived/importedfields/summary.cfg @@ -5,39 +5,71 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "b_ref_with_summary" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 159551552 classes[].name "mysummary" classes[].omitsummaryfeatures false classes[].fields[].name "a_ref" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "b_ref_with_summary" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "my_int_field" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "my_int_field" classes[].fields[].name "my_string_field" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "my_string_field" classes[].fields[].name "my_int_array_field" classes[].fields[].type "jsonstring" +classes[].fields[].command "attribute" +classes[].fields[].source "my_int_array_field" classes[].fields[].name "my_int_wset_field" classes[].fields[].type "jsonstring" +classes[].fields[].command "attribute" +classes[].fields[].source "my_int_wset_field" classes[].fields[].name "my_ancient_int_field" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "my_ancient_int_field" classes[].fields[].name "my_filtered_int_array_field" classes[].fields[].type "jsonstring" +classes[].fields[].command "matchedattributeelementsfilter" +classes[].fields[].source "my_int_array_field" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1274088866 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/indexswitches/summary.cfg b/config-model/src/test/derived/indexswitches/summary.cfg index a58d5da25e1..edc38aacf89 100644 --- a/config-model/src/test/derived/indexswitches/summary.cfg +++ b/config-model/src/test/derived/indexswitches/summary.cfg @@ -5,13 +5,25 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "source" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "descr" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/inheritance/summary.cfg b/config-model/src/test/derived/inheritance/summary.cfg index 6d30c0ad231..7615429bebf 100644 --- a/config-model/src/test/derived/inheritance/summary.cfg +++ b/config-model/src/test/derived/inheritance/summary.cfg @@ -5,22 +5,40 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "onlyfather" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1608562186 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "onlygrandparent" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "onlygrandparent" classes[].fields[].name "overridden" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "overridden" classes[].fields[].name "onlymother" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "onlymother" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/integerattributetostringindex/summary.cfg b/config-model/src/test/derived/integerattributetostringindex/summary.cfg index 9c2c8f4cd0e..87a6539de56 100644 --- a/config-model/src/test/derived/integerattributetostringindex/summary.cfg +++ b/config-model/src/test/derived/integerattributetostringindex/summary.cfg @@ -5,28 +5,52 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "attinx" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "attinx" classes[].fields[].name "artist" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "artist" classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1706878063 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "attinx" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "attinx" classes[].fields[].name "artist" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "artist" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg index b5ee816719e..a18ca43bbd7 100644 --- a/config-model/src/test/derived/map_attribute/summary.cfg +++ b/config-model/src/test/derived/map_attribute/summary.cfg @@ -5,11 +5,21 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "str_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "" classes[].fields[].name "int_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg index 997743389c6..c871c4bb97e 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -5,24 +5,44 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "str_elem_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "" classes[].fields[].name "int_elem_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "new_int_elem_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "copy" +classes[].fields[].source "int_elem_map" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1424421039 classes[].name "rename" classes[].omitsummaryfeatures false classes[].fields[].name "new_str_elem_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "attributecombiner" +classes[].fields[].source "str_elem_map" classes[].fields[].name "new_int_elem_map" classes[].fields[].type "jsonstring" +classes[].fields[].command "copy" +classes[].fields[].source "int_elem_map" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/mlr/summary.cfg b/config-model/src/test/derived/mlr/summary.cfg index 0345ec305ae..8087955a206 100644 --- a/config-model/src/test/derived/mlr/summary.cfg +++ b/config-model/src/test/derived/mlr/summary.cfg @@ -5,22 +5,40 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "b" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1944325986 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "ranklog" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "ranklog" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index c05cb43dade..17d7040bc78 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -5,192 +5,350 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "loc_pos" classes[].fields[].type "jsonstring" +classes[].fields[].command "geopos" +classes[].fields[].source "loc_pos_zcurve" classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "adynamic" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "adynamic" classes[].fields[].name "abolded" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "abolded" classes[].fields[].name "b" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "d" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "d" classes[].fields[].name "dynamice" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "dynamice" classes[].fields[].name "f" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "g" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "h" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "e" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "adynamic2" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "adynamic2" classes[].fields[].name "alltags" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "sometags" classes[].fields[].type "jsonstring" +classes[].fields[].command "matchedelementsfilter" +classes[].fields[].source "mytags" classes[].fields[].name "anotherb" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "abolded2" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "abolded2" classes[].fields[].name "aboldeddynamic" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "aboldeddynamic" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 783153771 classes[].name "third" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "adynamic" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "adynamic" classes[].fields[].name "d" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "d" classes[].fields[].name "e" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "f" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "g" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "h" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 815922035 classes[].name "attributesonly1" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1308077923 classes[].name "notattributesonly1" classes[].omitsummaryfeatures false classes[].fields[].name "adynamic" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "adynamic" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1609068631 classes[].name "anothernotattributesonly2" classes[].omitsummaryfeatures false classes[].fields[].name "adynamic2" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "adynamic2" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "alltags" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "sometags" classes[].fields[].type "jsonstring" +classes[].fields[].command "matchedelementsfilter" +classes[].fields[].source "mytags" classes[].fields[].name "anothera" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "anotherb" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 686755772 classes[].name "notattributesonly3" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "d" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "d" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1711750363 classes[].name "attributesonly2" classes[].omitsummaryfeatures false classes[].fields[].name "anotdynamic" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "adynamic" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "loc_position" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "loc_pos_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1510953467 classes[].name "attributesonly3" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "anotbolded" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "loc_pos_zcurve" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "loc_pos_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 923824943 classes[].name "notattributesonly4" classes[].omitsummaryfeatures false classes[].fields[].name "abolded2" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "abolded2" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 552611075 classes[].name "notattributesonly5" classes[].omitsummaryfeatures false classes[].fields[].name "aboldeddynamic" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "aboldeddynamic" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 146047714 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "loc_pos_zcurve" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "loc_pos_zcurve" classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 324773027 classes[].name "second" classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "a" classes[].fields[].name "adynamic" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "adynamic" classes[].fields[].name "c" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "c" classes[].fields[].name "f" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/music/summary.cfg b/config-model/src/test/derived/music/summary.cfg index 007c2ef3345..10bb238dca7 100644 --- a/config-model/src/test/derived/music/summary.cfg +++ b/config-model/src/test/derived/music/summary.cfg @@ -5,102 +5,200 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "bgndata" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "bgndata" classes[].fields[].name "sales" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "sales" classes[].fields[].name "pto" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "pto" classes[].fields[].name "mid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "mid" classes[].fields[].name "ew" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "ew" classes[].fields[].name "surl" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "userrate" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "pid" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "weight" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "weight" classes[].fields[].name "url" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "isbn" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "fmt" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "albumid" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "disp_song" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "song" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "song" classes[].fields[].name "pfrom" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "bgnpfrom" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "bgnpfrom" classes[].fields[].name "categories" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "data" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "numreview" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "bgnsellers" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "image" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "artist" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "artistspid" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "newestedition" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "newestedition" classes[].fields[].name "bgnpto" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "bgnpto" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "did" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "did" classes[].fields[].name "scorekey" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "cbid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "cbid" classes[].fields[].name "metalvalue" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "hiphopvalue" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "powermetalvalue" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "progvalue" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 2060710706 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "sales" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "sales" classes[].fields[].name "pto" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "pto" classes[].fields[].name "mid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "mid" classes[].fields[].name "weight" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "weight" classes[].fields[].name "bgnpfrom" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "bgnpfrom" classes[].fields[].name "newestedition" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "newestedition" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "did" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "did" classes[].fields[].name "cbid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "cbid" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/newrank/summary.cfg b/config-model/src/test/derived/newrank/summary.cfg index 7f77633602e..79e0aa2ce6c 100644 --- a/config-model/src/test/derived/newrank/summary.cfg +++ b/config-model/src/test/derived/newrank/summary.cfg @@ -5,96 +5,188 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "bgndata" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "bgndata" classes[].fields[].name "sales" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "sales" classes[].fields[].name "pto" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "pto" classes[].fields[].name "mid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "mid" classes[].fields[].name "ew" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "ew" classes[].fields[].name "surl" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "userrate" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "pid" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "weight" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "weight" classes[].fields[].name "url" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "isbn" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "fmt" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "albumid" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "disp_song" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "song" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "song" classes[].fields[].name "pfrom" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "bgnpfrom" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "bgnpfrom" classes[].fields[].name "categories" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "data" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "numreview" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "bgnsellers" classes[].fields[].type "integer" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "image" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "artist" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "artistspid" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "newestedition" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "newestedition" classes[].fields[].name "bgnpto" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "bgnpto" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "did" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "did" classes[].fields[].name "scorekey" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "scorekey" classes[].fields[].name "cbid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "cbid" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1606815285 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "sales" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "sales" classes[].fields[].name "pto" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "pto" classes[].fields[].name "mid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "mid" classes[].fields[].name "weight" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "weight" classes[].fields[].name "bgnpfrom" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "bgnpfrom" classes[].fields[].name "newestedition" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "newestedition" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "did" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "did" classes[].fields[].name "scorekey" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "scorekey" classes[].fields[].name "cbid" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "cbid" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg index 2c46031bdad..cda3a7df60f 100644 --- a/config-model/src/test/derived/position_nosummary/summary.cfg +++ b/config-model/src/test/derived/position_nosummary/summary.cfg @@ -5,16 +5,28 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1530141163 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "pos_zcurve" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "pos_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg index 7fda1ca0c05..ca26f898b1d 100644 --- a/config-model/src/test/derived/position_summary/summary.cfg +++ b/config-model/src/test/derived/position_summary/summary.cfg @@ -5,18 +5,32 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "pos" classes[].fields[].type "jsonstring" +classes[].fields[].command "geopos" +classes[].fields[].source "pos_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1530141163 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "pos_zcurve" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "pos_zcurve" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/predicate_attribute/summary.cfg b/config-model/src/test/derived/predicate_attribute/summary.cfg index d01ddcfcf2d..88a0828f9e1 100644 --- a/config-model/src/test/derived/predicate_attribute/summary.cfg +++ b/config-model/src/test/derived/predicate_attribute/summary.cfg @@ -5,16 +5,28 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "some_predicate_field" classes[].fields[].type "string" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1274088866 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/rankingexpression/summary.cfg b/config-model/src/test/derived/rankingexpression/summary.cfg index a6ba84ca7a9..0dc717c8993 100644 --- a/config-model/src/test/derived/rankingexpression/summary.cfg +++ b/config-model/src/test/derived/rankingexpression/summary.cfg @@ -5,40 +5,76 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "artist" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "surl" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1736696699 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "year" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "year" classes[].fields[].name "foo1" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "foo1" classes[].fields[].name "foo2" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "foo2" classes[].fields[].name "foo3" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "foo3" classes[].fields[].name "foo4" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "foo4" classes[].fields[].name "bar1" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "bar1" classes[].fields[].name "bar2" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "bar2" classes[].fields[].name "bar3" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "bar3" classes[].fields[].name "bar4" classes[].fields[].type "integer" +classes[].fields[].command "attribute" +classes[].fields[].source "bar4" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/ranktypes/summary.cfg b/config-model/src/test/derived/ranktypes/summary.cfg index b39f10f354b..5499349184b 100644 --- a/config-model/src/test/derived/ranktypes/summary.cfg +++ b/config-model/src/test/derived/ranktypes/summary.cfg @@ -5,11 +5,21 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "title" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "descr" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/reference_fields/summary.cfg b/config-model/src/test/derived/reference_fields/summary.cfg index 2bcabe81c3c..9e70d42d874 100644 --- a/config-model/src/test/derived/reference_fields/summary.cfg +++ b/config-model/src/test/derived/reference_fields/summary.cfg @@ -5,25 +5,43 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "campaign_ref" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 428144659 classes[].name "explicit_summary" classes[].omitsummaryfeatures false classes[].fields[].name "yet_another_ref" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1274088866 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/schemainheritance/summary.cfg b/config-model/src/test/derived/schemainheritance/summary.cfg index 0d54f8ceaf7..d774250f209 100644 --- a/config-model/src/test/derived/schemainheritance/summary.cfg +++ b/config-model/src/test/derived/schemainheritance/summary.cfg @@ -5,46 +5,82 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "parent_field" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "parent_field" classes[].fields[].name "child_field" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "child_field" classes[].fields[].name "pf1" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "cf1" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 2134223620 classes[].name "parent_summary" classes[].omitsummaryfeatures false classes[].fields[].name "pf1" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 524210908 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "parent_field" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "parent_field" classes[].fields[].name "child_field" classes[].fields[].type "longstring" +classes[].fields[].command "attribute" +classes[].fields[].source "child_field" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].id 1486475170 classes[].name "child_summary" classes[].omitsummaryfeatures false classes[].fields[].name "pf1" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "cf1" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/streamingstruct/summary.cfg b/config-model/src/test/derived/streamingstruct/summary.cfg index 4c44f7f38b2..7ed8cb3a192 100644 --- a/config-model/src/test/derived/streamingstruct/summary.cfg +++ b/config-model/src/test/derived/streamingstruct/summary.cfg @@ -5,50 +5,96 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "coupleof" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "anothersummaryfield" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "a" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "m" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "b" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "c" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "c2" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "c3" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "n" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "array1" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "array2" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "array3" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "subject" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "g" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "g" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "snippet" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "snippet" classes[].fields[].name "snippet2" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 109252281 classes[].name "summ" classes[].omitsummaryfeatures false classes[].fields[].name "snippet" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "snippet" classes[].fields[].name "snippet2" classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/streamingstructdefault/summary.cfg b/config-model/src/test/derived/streamingstructdefault/summary.cfg index 71b8c1371b3..e06f934554d 100644 --- a/config-model/src/test/derived/streamingstructdefault/summary.cfg +++ b/config-model/src/test/derived/streamingstructdefault/summary.cfg @@ -5,13 +5,25 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "sum1" classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "sum1" classes[].fields[].name "f1" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "f2" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg index a5b38684e52..444ab70d2a7 100644 --- a/config-model/src/test/derived/tensor/summary.cfg +++ b/config-model/src/test/derived/tensor/summary.cfg @@ -5,24 +5,44 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "f1" classes[].fields[].type "tensor" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "f3" classes[].fields[].type "tensor" +classes[].fields[].command "attribute" +classes[].fields[].source "f3" classes[].fields[].name "f4" classes[].fields[].type "tensor" +classes[].fields[].command "attribute" +classes[].fields[].source "f4" classes[].fields[].name "f5" classes[].fields[].type "tensor" +classes[].fields[].command "attribute" +classes[].fields[].source "f5" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 219619290 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "f6" classes[].fields[].type "float" +classes[].fields[].command "attribute" +classes[].fields[].source "f6" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg index 33af9d88630..6b2085d0975 100644 --- a/config-model/src/test/derived/types/summary.cfg +++ b/config-model/src/test/derived/types/summary.cfg @@ -5,42 +5,80 @@ classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "abyte" classes[].fields[].type "byte" +classes[].fields[].command "attribute" +classes[].fields[].source "abyte" classes[].fields[].name "along" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "along" classes[].fields[].name "abool" classes[].fields[].type "bool" +classes[].fields[].command "attribute" +classes[].fields[].source "abool" classes[].fields[].name "ashortfloat" classes[].fields[].type "float16" +classes[].fields[].command "attribute" +classes[].fields[].source "ashortfloat" classes[].fields[].name "tagfield" classes[].fields[].type "jsonstring" +classes[].fields[].command "attribute" +classes[].fields[].source "tagfield" classes[].fields[].name "stringmapfield" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "album0" classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" classes[].fields[].name "album1" classes[].fields[].type "jsonstring" +classes[].fields[].command "attribute" +classes[].fields[].source "album1" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" classes[].id 1027812395 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "other" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "other" classes[].fields[].name "abyte" classes[].fields[].type "byte" +classes[].fields[].command "attribute" +classes[].fields[].source "abyte" classes[].fields[].name "along" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "along" classes[].fields[].name "abool" classes[].fields[].type "bool" +classes[].fields[].command "attribute" +classes[].fields[].source "abool" classes[].fields[].name "ashortfloat" classes[].fields[].type "float16" +classes[].fields[].command "attribute" +classes[].fields[].source "ashortfloat" classes[].fields[].name "juletre" classes[].fields[].type "int64" +classes[].fields[].command "attribute" +classes[].fields[].source "juletre" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java index 06fa63707c0..b538d834df9 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java @@ -79,6 +79,43 @@ public class RankPropertiesTestCase extends AbstractSchemaTestCase { } @Test + public void testDefaultRankProperties() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry(), new TestProperties().setPhraseOptimization("split")); + builder.addSchema(joinLines( + "search test {", + " document test {", + " field a type string { ", + " indexing: index ", + " }", + " }", + " rank-profile a {", + " first-phase {", + " expression: a", + " }", + " }", + " rank-profile b {", + " first-phase {", + " expression: a", + " }", + " rank-properties {", + " query(a): 2000 ", + " }", + " }", + "}")); + builder.build(true); + Schema schema = builder.getSchema(); + List<RankProfile.RankProperty> props = rankProfileRegistry.get(schema, "a").getRankProperties(); + assertEquals(1, props.size()); + assertEquals(new RankProfile.RankProperty("vespa.matching.split_unpacking_iterators","true"), props.get(0)); + + props = rankProfileRegistry.get(schema, "b").getRankProperties(); + assertEquals(2, props.size()); + assertEquals(new RankProfile.RankProperty("vespa.matching.split_unpacking_iterators","true"), props.get(0)); + assertEquals(new RankProfile.RankProperty("query(a)","2000"), props.get(1)); + } + + @Test void testRankProfileMutate() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index a9a230c94a2..c83fc1ac63b 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -66,57 +66,19 @@ public class SummaryTestCase extends AbstractSchemaTestCase { assertEquals(13, summary.fields().size()); - field = fields.next(); - assertEquals("exactemento", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("exact", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("title", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("description", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("dyndesc", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("longdesc", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("longstat", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("dynlong", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("dyndesc2", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("measurement", field.getName()); - assertEquals(SummaryClassField.Type.INTEGER, field.getType()); - - field = fields.next(); - assertEquals("rankfeatures", field.getName()); - assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); - - field = fields.next(); - assertEquals("summaryfeatures", field.getName()); - assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); - - field = fields.next(); - assertEquals("documentid", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + assertSummaryField("exactemento", SummaryClassField.Type.LONGSTRING, fields.next()); + assertSummaryField("exact", SummaryClassField.Type.LONGSTRING, fields.next()); + assertSummaryField("title", SummaryClassField.Type.LONGSTRING, fields.next()); + assertSummaryField("description", SummaryClassField.Type.LONGSTRING, fields.next()); + assertSummaryField("dyndesc", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "dyndesc", fields.next()); + assertSummaryField("longdesc", SummaryClassField.Type.LONGSTRING, fields.next()); + assertSummaryField("longstat", SummaryClassField.Type.LONGSTRING, fields.next()); + assertSummaryField("dynlong", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "dynlong", fields.next()); + assertSummaryField("dyndesc2", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "dyndesc2", fields.next()); + assertSummaryField("measurement", SummaryClassField.Type.INTEGER, "attribute", "measurement", fields.next()); + assertSummaryField("rankfeatures", SummaryClassField.Type.FEATUREDATA, "rankfeatures", fields.next()); + assertSummaryField("summaryfeatures", SummaryClassField.Type.FEATUREDATA, "summaryfeatures", fields.next()); + assertSummaryField("documentid", SummaryClassField.Type.LONGSTRING, "documentid", fields.next()); } @Test @@ -132,6 +94,23 @@ public class SummaryTestCase extends AbstractSchemaTestCase { assertEquals(SummaryClassField.Type.LONGSTRING, myClass.fields().get("other_campaign_ref").getType()); } + private void assertSummaryField(String expName, SummaryClassField.Type expType, SummaryClassField field) { + assertSummaryField(expName, expType, "", "", field); + } + + private void assertSummaryField(String expName, SummaryClassField.Type expType, String expCommand, SummaryClassField field) { + assertSummaryField(expName, expType, expCommand, "", field); + } + + private void assertSummaryField(String expName, SummaryClassField.Type expType, + String expCommand, String expSource, SummaryClassField field) { + assertEquals(expName, field.getName()); + assertEquals(expType, field.getType()); + assertEquals(expCommand, field.getCommand()); + assertEquals(expSource, field.getSource()); + + } + private static Schema buildCampaignAdModel() throws ParseException { ApplicationBuilder builder = new ApplicationBuilder(); builder.addSchema("search campaign { document campaign {} }"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 4bf4cc71890..c71f3946937 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -41,6 +41,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "localhost"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0); assertEquals(0, config.myid()); + assertEquals("/opt/vespa/conf/zookeeper/tls.conf.json", config.vespaTlsConfigFile()); } @Test @@ -51,6 +52,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0, 1, 2); assertEquals(1, config.myid()); assertEquals("gz", config.snapshotMethod()); + assertEquals("/opt/vespa/conf/zookeeper/tls.conf.json", config.vespaTlsConfigFile()); } @Test @@ -60,6 +62,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); + assertEquals("/opt/vespa/conf/zookeeper/tls.conf.json", config.vespaTlsConfigFile()); } @Test @@ -71,6 +74,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); assertEquals("gz", config.snapshotMethod()); + assertEquals("", config.vespaTlsConfigFile()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index e8703f57fe3..c5baee01e28 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -187,7 +187,6 @@ public class DocprocBuilderTest extends DomBuilderTest { assertEquals(1536, jvm.minHeapsize()); assertEquals(1536, jvm.heapsize()); assertEquals(512, jvm.stacksize()); - assertTrue(qrStartConfig.ulimitv().isEmpty()); assertEquals(0, jvm.compressedClassSpaceSize()); } diff --git a/config/src/Doxyfile b/config/src/Doxyfile deleted file mode 100644 index 73231abd4ac..00000000000 --- a/config/src/Doxyfile +++ /dev/null @@ -1,1255 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.4.7 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = cloudconfig - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../../doc - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, -# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, -# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, -# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, -# Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce -# a new page for each member. If set to NO, the documentation of a member will -# be part of the file/class/namespace that contains it. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to -# include (a tag file for) the STL sources as input, then you should -# set this tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. -# func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. - -BUILTIN_STL_SUPPORT = NO - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. The default is NO. - -SHOW_DIRECTORIES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from the -# version control system). Doxygen will invoke the program by executing (via -# popen()) the command <command> <input-file>, where <command> is the value of -# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file -# provided by doxygen. Whatever the program writes to standard output -# is used as the file version. See the manual for examples. - -FILE_VERSION_FILTER = - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be abled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. - -WARN_NO_PARAMDOC = NO - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = config - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx -# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py - -FILE_PATTERNS = *.h \ - *.hpp \ - *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or -# directories that are symbolic links (a Unix filesystem feature) are excluded -# from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. Note that the wildcards are matched -# against the file with absolute path, so to exclude all test directories -# for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. Otherwise they will link to the documentstion. - -REFERENCES_LINK_SOURCE = YES - -# If the USE_HTAGS tag is set to YES then the references to source code -# will point to the HTML generated by the htags(1) tool instead of doxygen -# built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You -# will need version 4.8.6 or higher. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_DEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = IAM_DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse -# the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base -# or super classes. Setting the tag to NO turns the diagrams off. Note that -# this option is superseded by the HAVE_DOT option below. This is only a -# fallback. It is recommended to install and use dot, since it yields more -# powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for groups, showing the direct groups dependencies - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a caller dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable caller graphs for selected -# functions only using the \callergraph command. - -CALLER_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES -# then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include -# relations between the files in the directories. - -DIRECTORY_GRAPH = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes -# that lay further from the root node will be omitted. Note that setting this -# option to 1 or 2 may greatly reduce the computation time needed for large -# code bases. Also note that a graph may be further truncated if the graph's -# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH -# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), -# the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, which results in a white background. -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). - -DOT_TRANSPARENT = NO - -# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) -# support this, this feature is disabled by default. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def index 9b231d86c38..29438fda3b6 100644 --- a/configdefinitions/src/vespa/summary.def +++ b/configdefinitions/src/vespa/summary.def @@ -10,3 +10,8 @@ classes[].name string classes[].omitsummaryfeatures bool default=false classes[].fields[].name string classes[].fields[].type string +# The name of the command that is writing this field. +# See docsumconfig.cpp for all supported commands (DocsumFieldWriter implementations). +classes[].fields[].command string default="" +# The name of the source field used by the command. +classes[].fields[].source string default="" diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index a2c4ec3b2db..5cff46dd226 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -16,8 +16,6 @@ maxClientConnections int default=0 dataDir string default="var/zookeeper" clientPort int default=2181 -# TODO(bjorncs): remove setting - no longer in use -secureClientPort int default=2184 snapshotCount int default=50000 # Purge interval in hours @@ -45,13 +43,8 @@ server[].retired bool default=false # TODO: Consider setting this to false by default (and override where appropriate) trustEmptySnapshot bool default=true -# TLS options -# TODO(bjorncs): todo cleanup after migrating to unified Vespa TLS configuration -tlsForQuorumCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF -# TODO(bjorncs): todo cleanup after migrating to unified Vespa TLS configuration -tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF -# TODO(bjorncs): remove setting - no longer in use -jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" - dynamicReconfiguration bool default=false snapshotMethod string default="gz" + +# Uses default Vespa mTLS config if empty string +vespaTlsConfigFile string default="" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 69551c62840..48450716131 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -167,6 +167,7 @@ public class ModelContextImpl implements ModelContext { public static class FeatureFlags implements ModelContext.FeatureFlags { private final String queryDispatchPolicy; + private final String phraseOptimization; private final double defaultTermwiseLimit; private final boolean useThreePhaseUpdates; private final String feedSequencer; @@ -280,9 +281,11 @@ public class ModelContextImpl implements ModelContext { this.rpc_num_targets = flagValue(source, appId, version, Flags.RPC_NUM_TARGETS); this.rpc_events_before_wakeup = flagValue(source, appId, version, Flags.RPC_EVENTS_BEFORE_WAKEUP); this.queryDispatchPolicy = flagValue(source, appId, version, Flags.QUERY_DISPATCH_POLICY); + this.phraseOptimization = flagValue(source, appId, version, Flags.PHRASE_OPTIMIZATION); } - @Override public String queryDispatchPolicy() { return queryDispatchPolicy;} + @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } + @Override public String phraseOptimization() { return phraseOptimization; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @Override public String feedSequencerType() { return feedSequencer; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 2700f401a86..53dbda9c4d1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -35,25 +35,20 @@ public class HttpHandler extends ThreadedHttpRequestHandler { public HttpResponse handle(HttpRequest request) { log.log(Level.FINE, () -> request.getMethod() + " " + request.getUri().toString()); try { - switch (request.getMethod()) { - case POST: - return handlePOST(request); - case GET: - return handleGET(request); - case PUT: - return handlePUT(request); - case DELETE: - return handleDELETE(request); - default: - return createErrorResponse(request.getMethod()); - } + return switch (request.getMethod()) { + case POST -> handlePOST(request); + case GET -> handleGET(request); + case PUT -> handlePUT(request); + case DELETE -> handleDELETE(request); + default -> createErrorResponse(request.getMethod()); + }; } catch (NotFoundException | com.yahoo.vespa.config.server.NotFoundException e) { return HttpErrorResponse.notFoundError(getMessage(e, request)); } catch (ActivationConflictException e) { return HttpErrorResponse.conflictWhenActivating(getMessage(e, request)); } catch (InvalidApplicationException e) { return HttpErrorResponse.invalidApplicationPackage(getMessage(e, request)); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | UnsupportedOperationException e) { return HttpErrorResponse.badRequest(getMessage(e, request)); } catch (NodeAllocationException e) { return HttpErrorResponse.nodeAllocationFailure(getMessage(e, request)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 63ba8197960..69796e4d0f8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -301,7 +301,14 @@ public class SessionPreparer { (path, attr) -> attr.isRegularFile() && path.getFileName().toString().matches(".*\\.[Jj][Aa][Rr]"))) { paths.forEach(jarPath -> { try { - new BundleValidator().getPomXmlContent(logger, new JarFile(jarPath.toFile())); + new BundleValidator().getPomXmlContent(logger, new JarFile(jarPath.toFile())).ifPresent(pom -> { + try { + new ValidationProcessor().process(pom); + } + catch (IOException | TransformerException e) { + throw new RuntimeException(e); + } + }); } catch (ZipException e) { // ignore for tests } catch (IOException e) { diff --git a/container-core/src/main/java/com/yahoo/container/di/Container.java b/container-core/src/main/java/com/yahoo/container/di/Container.java index 1baf217da6b..dbf2ba4a9a8 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Container.java +++ b/container-core/src/main/java/com/yahoo/container/di/Container.java @@ -94,6 +94,14 @@ public class Container { } } + private void constructComponents(ComponentGraph graph) { + graph.nodes().forEach(n -> { + if (Thread.interrupted()) + throw new UncheckedInterruptedException("Interrupted while constructing component graph", true); + n.constructInstance(); + }); + } + private ComponentGraph waitForNewConfigGenAndCreateGraph( ComponentGraph graph, Injector fallbackInjector, boolean isInitializing, Collection<Bundle> obsoleteBundles) // NOTE: Return value { @@ -122,7 +130,7 @@ public class Container { Collection<Bundle> bundlesToRemove = installApplicationBundles(snapshot.configs()); obsoleteBundles.addAll(bundlesToRemove); - graph = createComponentsGraph(snapshot.configs(), getBootstrapGeneration(), fallbackInjector); + graph = createComponentGraph(snapshot.configs(), getBootstrapGeneration(), fallbackInjector); // Continues loop @@ -131,7 +139,7 @@ public class Container { } } log.log(FINE, () -> "Got components configs,\n" + configGenerationsString()); - return createAndConfigureComponentsGraph(snapshot.configs(), fallbackInjector); + return createAndConfigureComponentGraph(snapshot.configs(), fallbackInjector); } private long getBootstrapGeneration() { @@ -153,21 +161,13 @@ public class Container { throw new RuntimeException("Platform bundles are not allowed to change!\nOld: " + platformBundles + "\nNew: " + checkPlatformBundles); } - private ComponentGraph createAndConfigureComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> componentsConfigs, - Injector fallbackInjector) { - ComponentGraph componentGraph = createComponentsGraph(componentsConfigs, getComponentsGeneration(), fallbackInjector); + private ComponentGraph createAndConfigureComponentGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> componentsConfigs, + Injector fallbackInjector) { + ComponentGraph componentGraph = createComponentGraph(componentsConfigs, getComponentsGeneration(), fallbackInjector); componentGraph.setAvailableConfigs(componentsConfigs); return componentGraph; } - private void constructComponents(ComponentGraph graph) { - graph.nodes().forEach(n -> { - if (Thread.interrupted()) - throw new UncheckedInterruptedException("Interrupted while constructing component graph", true); - n.constructInstance(); - }); - } - private void deconstructFailedGraph(ComponentGraph currentGraph, ComponentGraph failedGraph) { Set<Object> currentComponents = Collections.newSetFromMap(new IdentityHashMap<>(currentGraph.size())); currentComponents.addAll(currentGraph.allConstructedComponentsAndProviders()); @@ -199,8 +199,8 @@ public class Container { return osgi.useApplicationBundles(applicationBundlesConfig.bundles()); } - private ComponentGraph createComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs, - long generation, Injector fallbackInjector) { + private ComponentGraph createComponentGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs, + long generation, Injector fallbackInjector) { previousConfigGeneration = generation; ComponentGraph graph = new ComponentGraph(generation); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 18490765576..ac50cbbb518 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -91,8 +91,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { .map(detectorConnFactory -> detectorConnFactory.getBean(SslConnectionFactory.class))) .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory()) .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); - ConnectorConfig.ProxyProtocol proxyProtocolCfg = targetConnector.connectorConfig().proxyProtocol(); - boolean proxyProtocol = proxyProtocolCfg.enabled() && !proxyProtocolCfg.mixedMode(); + boolean proxyProtocol = targetConnector.connectorConfig().proxyProtocol().enabled(); return new ProxyTarget(targetPort, clientTimeout,handlerTimeout, cacheExpiry, sslContextFactory, proxyProtocol); } diff --git a/container-core/src/main/java/com/yahoo/processing/request/CloneHelper.java b/container-core/src/main/java/com/yahoo/processing/request/CloneHelper.java index 8ff54ab6728..64a356fbf2a 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CloneHelper.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CloneHelper.java @@ -8,7 +8,6 @@ import com.yahoo.lang.PublicCloneable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.logging.Logger; import java.util.LinkedList; import java.util.ArrayList; @@ -35,9 +34,7 @@ public class CloneHelper { private static final Logger log = Logger.getLogger(CloneHelper.class.getName()); private static final MethodCache cloneMethodCache = new MethodCache("clone"); - /** - * Clones this object if it is clonable, and the clone is public. Returns null if not - */ + /** Clones this object if it is clonable, and the clone is public. Returns null if not. */ public final Object clone(Object object) { if (object == null) return null; if ( ! (object instanceof Cloneable)) return null; @@ -115,24 +112,22 @@ public class CloneHelper { Method cloneMethod = cloneMethodCache.get(object); if (cloneMethod == null) { log.warning("'" + object + "' of class " + object.getClass() + - " is Cloneable, but has no clone method - will use the same instance in all requests"); + " is Cloneable, but has no clone method - will use the same instance in all requests"); return null; } return cloneMethod.invoke(object); } catch (IllegalAccessException e) { log.warning("'" + object + "' of class " + object.getClass() + - " is Cloneable, but clone method cannot be accessed - will use the same instance in all requests"); + " is Cloneable, but clone method cannot be accessed - will use the same instance in all requests"); return null; } catch (InvocationTargetException e) { throw new RuntimeException("Exception cloning '" + object + "'", e); } } - /** - * Clones a map by deep cloning each value which is cloneable and shallow copying all other values. - */ + /** Clones a map by deep cloning each value which is cloneable and shallow copying all other values. */ public Map<CompoundName, Object> cloneMap(Map<CompoundName, Object> map) { - Map<CompoundName, Object> cloneMap = new HashMap<>(map.size()); + Map<CompoundName, Object> cloneMap = new HashMap<>((int)(map.size()/0.75) + 1); for (Map.Entry<CompoundName, Object> entry : map.entrySet()) { Object cloneValue = clone(entry.getValue()); if (cloneValue == null) diff --git a/container-core/src/main/java/com/yahoo/processing/request/properties/PropertyMap.java b/container-core/src/main/java/com/yahoo/processing/request/properties/PropertyMap.java index f149c177b47..adaac4216b9 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/properties/PropertyMap.java +++ b/container-core/src/main/java/com/yahoo/processing/request/properties/PropertyMap.java @@ -22,9 +22,6 @@ import java.util.Map; */ public class PropertyMap extends Properties { - /** - * The properties of this - */ private Map<CompoundName, Object> properties = new HashMap<>(); public void set(CompoundName name, Object value, Map<String, String> context) { diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index b4ce6a03ba8..97f74bd8d26 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -60,7 +60,7 @@ getconfig() { qrstartcfg="`$VESPA_HOME/bin/vespa-get-config -l -w 10 -n search.config.qr-start -i ${VESPA_CONFIG_ID}`" ;; esac - cmds=`echo "$qrstartcfg" | perl -ne 's/^(\w+)\.(\w+) (.*)/$1_$2=$3/ && print'` + cmds=`echo "$qrstartcfg" | sed -n 's/^\([^. ]*\)[.]/\1_/;s/ /=/p'` eval "$cmds" set +e } diff --git a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def index c58f9944d61..e0b7d15085b 100644 --- a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def +++ b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def @@ -40,9 +40,6 @@ jvm.availableProcessors int default=0 restart ## Extra environment variables qrs.env string default="" restart -## Set ulimit -v in start script? this is mainly a safeguard against JNI stuff leaking memory. -ulimitv string default="" restart - ## Extra class path entries, forwarded to vespa-start-container_daemon. Overrides the corresponding env setting. jdisc.classpath_extra string default="" restart diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java index d80b2201810..fdcda9e5403 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.component.Version; +import java.time.Duration; import java.time.Instant; import java.util.Objects; @@ -11,16 +12,12 @@ import java.util.Objects; * * @author mpolden */ -public class OsRelease { +public record OsRelease(Version version, Tag tag, Instant taggedAt) { - private final Version version; - private final Tag tag; - private final Instant taggedAt; - - public OsRelease(Version version, Tag tag, Instant taggedAt) { - this.version = Objects.requireNonNull(version); - this.tag = Objects.requireNonNull(tag); - this.taggedAt = Objects.requireNonNull(taggedAt); + public OsRelease { + Objects.requireNonNull(version); + Objects.requireNonNull(tag); + Objects.requireNonNull(taggedAt); } /** The version number */ @@ -38,22 +35,14 @@ public class OsRelease { return taggedAt; } - @Override - public String toString() { - return "os release " + version + ", tagged " + tag + " at " + taggedAt; + /** Returns the age of this at given instant */ + public Duration age(Instant instant) { + return Duration.between(taggedAt, instant); } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - OsRelease osRelease = (OsRelease) o; - return version.equals(osRelease.version) && tag == osRelease.tag && taggedAt.equals(osRelease.taggedAt); - } - - @Override - public int hashCode() { - return Objects.hash(version, tag, taggedAt); + public String toString() { + return "os release " + version + ", tagged " + tag + " at " + taggedAt; } /** Known release tags */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 32d84d9791d..a9d67c2d78a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -71,10 +71,6 @@ public enum RoleDefinition { Policy.applicationManager, Policy.keyRevokal, Policy.paymentInstrumentRead, - Policy.paymentInstrumentUpdate, - Policy.paymentInstrumentDelete, - Policy.paymentInstrumentCreate, - Policy.planUpdate, Policy.billingInformationRead, Policy.accessRequests ), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 9dac13482e0..a4ce45f44ea 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -166,9 +166,9 @@ public class RoleTest { Role admin = Role.administrator(TenantName.from("t1")); assertTrue(publicCdEnforcer.allows(admin, Action.read, paymentInstrumentUri)); - assertTrue(publicCdEnforcer.allows(admin, Action.delete, paymentInstrumentUri)); - assertTrue(publicCdEnforcer.allows(admin, Action.update, tenantPaymentInstrumentUri)); - assertTrue(publicCdEnforcer.allows(admin, Action.read, tokenUri)); + assertFalse(publicCdEnforcer.allows(admin, Action.delete, paymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(admin, Action.update, tenantPaymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(admin, Action.read, tokenUri)); } @Test @@ -204,7 +204,6 @@ public class RoleTest { .assertAction(operator) .assertAction(reader) .assertAction(developer) - .assertAction(admin, Action.read) .assertAction(otherAdmin); tester.on("/billing/v1/tenant/t1/instrument") @@ -212,7 +211,7 @@ public class RoleTest { .assertAction(operator, Action.read) .assertAction(reader, Action.read, Action.delete) .assertAction(developer, Action.read, Action.delete) - .assertAction(admin, Action.read, Action.update, Action.delete) + .assertAction(admin, Action.read) .assertAction(otherAdmin); tester.on("/billing/v1/tenant/t1/instrument/i1") @@ -220,7 +219,7 @@ public class RoleTest { .assertAction(operator, Action.read) .assertAction(reader, Action.read, Action.delete) .assertAction(developer, Action.read, Action.delete) - .assertAction(admin, Action.read, Action.update, Action.delete) + .assertAction(admin, Action.read) .assertAction(otherAdmin); tester.on("/billing/v1/tenant/t1/billing") @@ -236,7 +235,7 @@ public class RoleTest { .assertAction(operator, Action.read) .assertAction(reader) .assertAction(developer) - .assertAction(admin, Action.update) + .assertAction(admin) .assertAction(otherAdmin); tester.on("/billing/v1/tenant/t1/collection") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index c86f79ce188..93ef6d29450 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -52,6 +52,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { Run last = controller().jobController().last(job).get(); Versions target = new Versions(targetPlatform, last.versions().targetRevision(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetRevision())); + if ( ! last.hasEnded()) continue; if (application.revisions().get(last.versions().targetRevision()).compileVersion() .map(version -> controller().applications().versionCompatibility(instance.id()).refuse(version, target.targetPlatform())) .orElse(false)) continue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 6f4a8429c74..30a98cbfacd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -79,15 +79,21 @@ public class OsUpgradeScheduler extends ControllerMaintainer { return hourOfDay >= startHour && hourOfDay <= 12 && dayOfWeek < 5; } - /** Returns the earliest time an upgrade can be scheduled on the day of instant, in given system */ + /** Returns the earliest time, at or after instant, an upgrade can be scheduled */ private static Instant schedulingInstant(Instant instant, SystemName system) { - instant = instant.truncatedTo(ChronoUnit.DAYS); + ChronoUnit schedulingResolution = ChronoUnit.HOURS; while (!canTriggerAt(instant, system.isCd())) { - instant = instant.plus(Duration.ofHours(1)); + instant = instant.truncatedTo(schedulingResolution) + .plus(schedulingResolution.getDuration()); } return instant; } + /** Returns the remaining cool-down period relative to releaseAge */ + private static Duration remainingCooldownOf(Duration cooldown, Duration releaseAge) { + return releaseAge.compareTo(cooldown) < 0 ? cooldown.minus(releaseAge) : Duration.ZERO; + } + private interface Release { /** The pending change for this release at given instant, if any */ @@ -123,7 +129,8 @@ public class OsUpgradeScheduler extends ControllerMaintainer { public Optional<Change> change(Version currentVersion, Instant instant) { OsRelease release = artifactRepository.osRelease(currentVersion.getMajor(), tag()); if (!release.version().isAfter(currentVersion)) return Optional.empty(); - Instant scheduleAt = schedulingInstant(release.taggedAt().plus(cooldown()), system); + Duration cooldown = remainingCooldownOf(cooldown(), release.age(instant)); + Instant scheduleAt = schedulingInstant(instant.plus(cooldown), system); return Optional.of(new Change(release.version(), Duration.ZERO, scheduleAt)); } @@ -165,24 +172,19 @@ public class OsUpgradeScheduler extends ControllerMaintainer { predicatedInstant = predicatedInstant.plus(Duration.ofDays(1)); version = findVersion(predicatedInstant, currentVersion); } - Duration cooldown = remainingCooldownAt(instant, version); + Duration cooldown = remainingCooldownOf(cooldown(), version.age(instant)); Instant schedulingInstant = schedulingInstant(instant.plus(cooldown), system); return Optional.of(new Change(version.version(), upgradeBudget(), schedulingInstant)); } - private Duration upgradeBudget() { - return system.isCd() ? Duration.ZERO : Duration.ofDays(14); - } - - private Duration remainingCooldownAt(Instant instant, CalendarVersion version) { - Duration minAge = system.isCd() + private Duration cooldown() { + return system.isCd() ? Duration.ofDays(1) // CD: Give new releases some time to propagate : Duration.ofDays(7 - RELEASE_DAY.ordinal()); // non-CD: Wait until start of the following week - Duration age = version.age(instant); - if (age.compareTo(minAge) < 0) { - return minAge.minus(age); - } - return Duration.ZERO; + } + + private Duration upgradeBudget() { + return system.isCd() ? Duration.ZERO : Duration.ofDays(14); } /** Find the most recent version available according to the scheduling step, relative to now */ @@ -211,10 +213,9 @@ public class OsUpgradeScheduler extends ControllerMaintainer { date); } - /** Returns the age of this at given instant, in whole days */ + /** Returns the age of this at given instant */ private Duration age(Instant instant) { - return Duration.between(date.atStartOfDay().toInstant(ZoneOffset.UTC), - instant.truncatedTo(ChronoUnit.DAYS)); + return Duration.between(date.atStartOfDay().toInstant(ZoneOffset.UTC), instant); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponses.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponses.java new file mode 100644 index 00000000000..4c4633df0ec --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponses.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.ErrorResponse; + +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Helper class for creating error responses. + * + * @author mpolden + */ +public class ErrorResponses { + + private ErrorResponses() {} + + /** + * Returns a response for a failing request containing an unique request ID. Details of the error are logged through + * given logger. + */ + public static ErrorResponse logThrowing(HttpRequest request, Logger logger, Throwable t) { + String requestId = UUID.randomUUID().toString(); + logger.log(Level.SEVERE, "Unexpected error handling '" + request.getUri() + "' (request ID: " + + requestId + ")", t); + return ErrorResponse.internalServerError("Unexpected error occurred (request ID: " + requestId + ")"); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index bb38f7eff0e..0cbd6b61bf8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -101,6 +101,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; @@ -226,13 +227,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return switch (e.code()) { case NOT_FOUND: yield ErrorResponse.notFoundError(Exceptions.toMessageString(e)); case ACTIVATION_CONFLICT: yield new ErrorResponse(CONFLICT, e.code().name(), Exceptions.toMessageString(e)); - case INTERNAL_SERVER_ERROR: yield ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + case INTERNAL_SERVER_ERROR: yield ErrorResponses.logThrowing(request, log, e); default: yield new ErrorResponse(BAD_REQUEST, e.code().name(), Exceptions.toMessageString(e)); }; } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } @@ -1051,7 +1051,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { SlimeUtils.copyObject(responseSlime.get(), responseResultCursor); return new SlimeJsonResponse(responseRoot); } catch (JsonParseException e) { - return ErrorResponse.internalServerError(response); + return ErrorResponses.logThrowing(request, log, e); } } @@ -2742,7 +2742,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static Principal requireUserPrincipal(HttpRequest request) { Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new RestApiException.InternalServerError("Expected a user principal"); + if (principal == null) throw new IllegalArgumentException("Expected a user principal"); return principal; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index d45e69f781b..bb55b2e8fcf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; @@ -43,7 +44,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; -import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -76,25 +76,18 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { return ErrorResponse.unauthorized("Must be authenticated to use this API"); Path path = new Path(request.getUri()); - switch (request.getMethod()) { - case GET: - return handleGET(request, path, userId.get()); - case PATCH: - return handlePATCH(request, path, userId.get()); - case DELETE: - return handleDELETE(path, userId.get()); - case POST: - return handlePOST(path, request, userId.get()); - default: - return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } + return switch (request.getMethod()) { + case GET -> handleGET(request, path, userId.get()); + case PATCH -> handlePATCH(request, path, userId.get()); + case DELETE -> handleDELETE(path, userId.get()); + case POST -> handlePOST(path, request, userId.get()); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (Exception e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - // Don't expose internal billing details in error message to user - return ErrorResponse.internalServerError("Internal problem while handling billing API request"); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 4532e0c2c18..8722e588fa7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -14,7 +13,6 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.Type; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; @@ -27,7 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.application.QuotaUsage; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -39,11 +37,15 @@ import java.time.format.DateTimeFormatter; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; /** * @author ogronnesby */ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandlerV2> { + + private static final Logger log = Logger.getLogger(BillingApiHandlerV2.class.getName()); + private static final String[] CSV_INVOICE_HEADER = new String[]{ "ID", "Tenant", "From", "To", "CpuHours", "MemoryHours", "DiskHours", "Cpu", "Memory", "Disk", "Additional" }; private final ApplicationController applications; @@ -85,7 +87,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .post(Slime.class, self::createBill)) .addRoute(RestApi.route("/billing/v2/accountant/plans") .get(self::plans)) - .addExceptionMapper(RuntimeException.class, (__, e) -> ErrorResponse.internalServerError(e.getMessage())) + .addExceptionMapper(RuntimeException.class, (c, e) -> ErrorResponses.logThrowing(c.request(), log, e)) .build(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index a3b77e22f1d..813d2b8d3e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -23,13 +23,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.maintenance.ChangeManagementAssessor; import com.yahoo.vespa.hosted.controller.persistence.ChangeRequestSerializer; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.logging.Level; import java.util.stream.Collectors; public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { @@ -61,8 +61,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 467b0c094cc..e764fed4653 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -17,11 +17,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.net.URI; import java.util.List; -import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -69,9 +69,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "', " - + Exceptions.toMessageString(e)); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 776fcbfd03b..9278d030db6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; @@ -38,7 +39,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Scanner; import java.util.function.Function; -import java.util.logging.Level; /** * This implements the controller/v1 API which provides operators with information about, @@ -73,8 +73,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index 9b400fdfb78..74a28276c79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.io.IOException; @@ -25,7 +26,6 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import java.util.function.Supplier; -import java.util.logging.Level; import java.util.logging.Logger; import static java.nio.charset.StandardCharsets.UTF_8; @@ -59,8 +59,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { } catch (IllegalArgumentException|IllegalStateException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java index ab8b6a1d26f..336352e931a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/CliApiHandler.java @@ -10,10 +10,9 @@ import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; -import java.util.logging.Level; - /** * This handler implements the /cli/v1/ API. The API allows Vespa CLI to retrieve information about the system, without * authorization. One example of such information is the minimum Vespa CLI version supported by our APIs. @@ -44,8 +43,7 @@ public class CliApiHandler extends ThreadedHttpRequestHandler { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 7e43de9f274..1bf2f78f866 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -31,7 +32,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; -import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -60,18 +60,17 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { - switch (request.getMethod()) { - case GET: return handleGET(request); - case OPTIONS: return handleOPTIONS(); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } + return switch (request.getMethod()) { + case GET -> handleGET(request); + case OPTIONS -> handleOPTIONS(); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java index f9f3025837d..3c0ec666415 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.TenantRole; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.io.IOException; @@ -28,7 +29,6 @@ import java.io.OutputStream; import java.util.EnumSet; import java.util.Optional; import java.util.Set; -import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -63,19 +63,18 @@ public class HorizonApiHandler extends ThreadedHttpRequestHandler { return ErrorResponse.forbidden("No tenant with enabled metrics view"); try { - switch (request.getMethod()) { - case GET: return get(request); - case POST: return post(request, authorizedTenants, operator); - case PUT: return put(request); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } + return switch (request.getMethod()) { + case GET -> get(request); + case POST -> post(request, authorizedTenants, operator); + case PUT -> put(request); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError("An unexpected error occurred"); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java index d6a0e785b43..2211e83c51a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriter.java @@ -9,9 +9,11 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import java.io.IOException; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * @author valerijf @@ -24,21 +26,21 @@ public class TsdbQueryRewriter { JsonNode root = mapper.readTree(data); requireLegalType(root); getField(root, "executionGraph", ArrayNode.class) - .ifPresent(graph -> rewriteQueryGraph(graph, authorizedTenants, operator, systemName)); + .ifPresent(graph -> rewriteQueryGraph(root, graph, authorizedTenants, operator, systemName)); getField(root, "filters", ArrayNode.class) .ifPresent(filters -> rewriteFilters(filters, authorizedTenants, operator, systemName)); getField(root, "queries", ArrayNode.class) - .ifPresent(graph -> rewriteQueryGraph(graph, authorizedTenants, operator, systemName)); + .ifPresent(graph -> rewriteQueryGraph(root, graph, authorizedTenants, operator, systemName)); return mapper.writeValueAsBytes(root); } - private static void rewriteQueryGraph(ArrayNode executionGraph, Set<TenantName> tenantNames, boolean operator, SystemName systemName) { + private static void rewriteQueryGraph(JsonNode root, ArrayNode executionGraph, Set<TenantName> tenantNames, boolean operator, SystemName systemName) { for (int i = 0; i < executionGraph.size(); i++) { JsonNode execution = executionGraph.get(i); // Will be handled by rewriteFilters() - if (execution.has("filterId")) continue; + if (execution.has("filterId") && filterExists(root, execution.get("filterId").asText())) continue; rewriteFilter((ObjectNode) execution, tenantNames, operator, systemName); } @@ -80,6 +82,16 @@ public class TsdbQueryRewriter { } } + private static boolean filterExists(JsonNode root, String filterId) { + return getField(root, "filters", ArrayNode.class).stream() + .flatMap(filters -> IntStream.range(0, filters.size()) + .mapToObj(i -> filters.get(i).get("id"))) + .filter(Objects::nonNull) + .filter(JsonNode::isTextual) + .map(JsonNode::asText) + .anyMatch(filterId::equals); + } + private static void requireLegalType(JsonNode root) { Optional.ofNullable(root.get("type")) .map(JsonNode::asText) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 0e764b98514..77d0e799101 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; import com.yahoo.vespa.hosted.controller.maintenance.OsUpgradeScheduler; import com.yahoo.vespa.hosted.controller.maintenance.OsUpgradeScheduler.Change; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.yolean.Exceptions; @@ -37,7 +38,6 @@ import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.function.Function; -import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -71,8 +71,7 @@ public class OsApiHandler extends AuditLoggingRequestHandler { } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 082b11af351..2ecd63546e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; @@ -38,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -69,8 +69,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index ed27ffad978..e9b087690ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -12,9 +12,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvi import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import java.util.concurrent.Executor; -import java.util.logging.Level; /** * Handler implementation for '/system-flags/v1', an API for controlling system-wide feature flags @@ -38,12 +38,10 @@ public class SystemFlagsHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - switch (request.getMethod()) { - case PUT: - return put(request); - default: - return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - } + return switch (request.getMethod()) { + case PUT -> put(request); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + }; } private HttpResponse put(HttpRequest request) { @@ -63,9 +61,7 @@ public class SystemFlagsHandler extends ThreadedHttpRequestHandler { SystemFlagsDeployResult result = deployer.deployFlags(archive, dryRun); return new JacksonJsonResponse<>(200, result.toWire()); } catch (Exception e) { - String errorMessage = "System flags deploy failed: " + e.getMessage(); - log.log(Level.SEVERE, errorMessage, e); - return ErrorResponse.internalServerError(errorMessage); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index a407e5aa211..9cced2b8159 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -8,6 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; @@ -27,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.api.role.TenantRole; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; @@ -51,7 +52,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -95,8 +95,7 @@ public class UserApiHandler extends ThreadedHttpRequestHandler { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index e2c20929be5..7978e64482b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -14,11 +14,11 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.util.Comparator; import java.util.List; -import java.util.logging.Level; /** * Read-only REST API that provides information about zones in hosted Vespa (version 1) @@ -38,18 +38,14 @@ public class ZoneApiHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { - switch (request.getMethod()) { - case GET: - return get(request); - default: - return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - } + return switch (request.getMethod()) { + case GET -> get(request); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "', " - + Exceptions.toMessageString(e)); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index e3f1e9b5f94..c9bbdc2c005 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -18,10 +18,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; -import java.util.logging.Level; - /** * REST API for proxying requests to config servers in a given zone (version 2). * @@ -45,23 +44,15 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { @Override public HttpResponse auditAndHandle(HttpRequest request) { try { - switch (request.getMethod()) { - case GET: - return get(request); - case POST: - case PUT: - case DELETE: - case PATCH: - return proxy(request); - default: - return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); - } + return switch (request.getMethod()) { + case GET -> get(request); + case POST, PUT, DELETE, PATCH -> proxy(request); + default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + }; } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "', " - + Exceptions.toMessageString(e)); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + return ErrorResponses.logThrowing(request, log, e); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index a41caeffa19..146135491a3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.OptionalInt; import java.util.StringJoiner; import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -364,11 +366,15 @@ public class ApplicationPackageBuilder { return new SimpleDateFormat("yyyy-MM-dd").format(Date.from(instant)); } - public static ApplicationPackage fromDeploymentXml(String deploymentXml) { - return fromDeploymentXml(deploymentXml, "6.1"); + public static ApplicationPackage fromDeploymentXml(String deploymentXml, ValidationId... overrides) { + return fromDeploymentXml(deploymentXml, "6.1", overrides); } public static ApplicationPackage fromDeploymentXml(String deploymentXml, String compileVersion) { + return fromDeploymentXml(deploymentXml, compileVersion, new ValidationId[0]); + } + + public static ApplicationPackage fromDeploymentXml(String deploymentXml, String compileVersion, ValidationId... overrides) { ByteArrayOutputStream zip = new ByteArrayOutputStream(); try (ZipOutputStream out = new ZipOutputStream(zip)) { out.putNextEntry(new ZipEntry("deployment.xml")); @@ -377,6 +383,14 @@ public class ApplicationPackageBuilder { out.putNextEntry(new ZipEntry("build-meta.json")); out.write(buildMeta(Version.fromString(compileVersion))); out.closeEntry(); + if (overrides.length > 0) { + out.putNextEntry(new ZipEntry("validation-overrides.xml")); + String override = "<allow until='" + asIso8601Date(Instant.now().plus(Duration.ofDays(28))) + "'>%s</allow>"; + out.write(("<validation-overrides version='1.0'>\n" + + Arrays.stream(overrides).map(ValidationId::value).map(override::formatted).collect(Collectors.joining("\n")) + + "</validation-overrides>\n").getBytes(UTF_8)); + out.closeEntry(); + } } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 93ebdcf3171..f39eb6e161d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -21,12 +21,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVers import java.net.URI; import java.time.Duration; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -36,12 +36,12 @@ import java.util.stream.Collectors; */ public class NodeRepositoryMock implements NodeRepository { - private final Map<ZoneId, Map<HostName, Node>> nodeRepository = new HashMap<>(); - private final Map<ZoneId, Map<ApplicationId, Application>> applications = new HashMap<>(); - private final Map<ZoneId, TargetVersions> targetVersions = new HashMap<>(); - private final Map<Integer, Duration> osUpgradeBudgets = new HashMap<>(); - private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new HashMap<>(); - private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new HashMap<>(); + private final Map<ZoneId, Map<HostName, Node>> nodeRepository = new ConcurrentHashMap<>(); + private final Map<ZoneId, Map<ApplicationId, Application>> applications = new ConcurrentHashMap<>(); + private final Map<ZoneId, TargetVersions> targetVersions = new ConcurrentHashMap<>(); + private final Map<Integer, Duration> osUpgradeBudgets = new ConcurrentHashMap<>(); + private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new ConcurrentHashMap<>(); + private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new ConcurrentHashMap<>(); private boolean allowPatching = true; private boolean hasSpareCapacity = false; @@ -117,7 +117,7 @@ public class NodeRepositoryMock implements NodeRepository { @Override public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) { - archiveUris.computeIfAbsent(zone, z -> new HashMap<>()).put(tenantName, archiveUri); + archiveUris.computeIfAbsent(zone, z -> new ConcurrentHashMap<>()).put(tenantName, archiveUri); } @Override @@ -213,7 +213,7 @@ public class NodeRepositoryMock implements NodeRepository { /** Add or update given nodes in zone */ public void putNodes(ZoneId zone, List<Node> nodes) { - Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); + Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new ConcurrentHashMap<>()); for (var node : nodes) { zoneNodes.put(node.hostname(), node); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 016db28c2aa..39264286e44 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -131,7 +131,7 @@ public class OsUpgradeSchedulerTest { void schedule_stable_release() { ControllerTester tester = new ControllerTester(); OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); - Instant t0 = Instant.parse("2021-06-21T06:00:00.00Z"); // Outside trigger period + Instant t0 = Instant.parse("2021-06-22T00:42:12.00Z"); // Outside trigger period tester.clock().setInstant(t0); // Set initial target @@ -139,13 +139,15 @@ public class OsUpgradeSchedulerTest { Version version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, Duration.ZERO, false); - // Stable release is scheduled once trigger period opens + // Stable release (tagged outside trigger period) is scheduled once trigger period opens Version version1 = Version.fromString("8.1"); tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable, - tester.clock().instant())); + Instant.parse("2021-06-21T23:59:00.00Z"))); scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester); - assertEquals(version1, scheduler.changeIn(cloud).get().version(), "Change available"); - scheduleUpgradeAfter(Duration.ofHours(1), version1, scheduler, tester); // Inside trigger period + OsUpgradeScheduler.Change nextChange = scheduler.changeIn(cloud).get(); + assertEquals(version1, nextChange.version()); + assertEquals("2021-06-22T07:00:00", formatInstant(nextChange.scheduleAt())); + scheduleUpgradeAfter(Duration.ofHours(7), version1, scheduler, tester); // Inside trigger period // A newer version is triggered manually Version version3 = Version.fromString("8.3"); @@ -160,7 +162,7 @@ public class OsUpgradeSchedulerTest { void schedule_latest_release_in_cd() { ControllerTester tester = new ControllerTester(SystemName.cd); OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); - Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period + Instant t0 = Instant.parse("2021-06-21T07:05:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); // Set initial target @@ -172,7 +174,9 @@ public class OsUpgradeSchedulerTest { Version version1 = Version.fromString("8.1"); tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest, tester.clock().instant())); - assertEquals(version1, scheduler.changeIn(cloud).get().version(), "Change available"); + assertEquals(version1, scheduler.changeIn(cloud).get().version()); + assertEquals("2021-06-22T07:05:00", formatInstant(scheduler.changeIn(cloud).get().scheduleAt()), + "Not valid until cool-down period passes"); scheduleUpgradeAfter(Duration.ZERO, version0, scheduler, tester); // Cooldown period passes and latest release is scheduled diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index a927439de1c..5fe44038d73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -29,33 +29,35 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { @Override protected String variablePartXml() { - return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + - - " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + - " <binding>http://*/application/v4/*</binding>\n" + - " </handler>\n" + - - " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + - " <binding>http://*/zone/v1</binding>\n" + - " <binding>http://*/zone/v1/*</binding>\n" + - " </handler>\n" + - - " <http>\n" + - " <server id='default' port='8080' />\n" + - " <filtering>\n" + - " <request-chain id='default'>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n"; + return """ + <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/> + <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/> + + <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'> + <binding>http://localhost/application/v4/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'> + <binding>http://localhost/zone/v1</binding> + <binding>http://localhost/zone/v1/*</binding> + </handler> + + <http> + <server id='default' port='8080' /> + <filtering> + <request-chain id='default'> + <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/> + <binding>http://localhost/*</binding> + </request-chain> + </filtering> + </http> + """; } - protected static final String accessDenied = "{\n" + - " \"code\" : 403,\n" + - " \"message\" : \"Access denied\"\n" + - "}"; + protected static final String accessDenied = """ + { + "code" : 403, + "message" : "Access denied" + }"""; protected RequestBuilder request(String path) { return new RequestBuilder(path, Request.Method.GET); } protected RequestBuilder request(String path, Request.Method method) { return new RequestBuilder(path, method); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 404dcca87c0..a410221f026 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -45,73 +45,77 @@ public class ControllerContainerTest { public void stopContainer() { container.close(); } private String controllerServicesXml() { - return "<container version='1.0'>\n" + - " <config name=\"container.handler.threadpool\">\n" + - " <maxthreads>10</maxthreads>\n" + - " </config> \n" + - " <config name=\"cloud.config.configserver\">\n" + - " <system>" + system().value() + "</system>\n" + - " </config> \n" + - " <config name=\"vespa.hosted.rotation.config.rotations\">\n" + - " <rotations>\n" + - " <item key=\"rotation-id-1\">rotation-fqdn-1</item>\n" + - " <item key=\"rotation-id-2\">rotation-fqdn-2</item>\n" + - " <item key=\"rotation-id-3\">rotation-fqdn-3</item>\n" + - " <item key=\"rotation-id-4\">rotation-fqdn-4</item>\n" + - " <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" + - " </rotations>\n" + - " </config>\n" + - " " + - "<accesslog type='disabled'/>\n" + - " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + - " <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/>\n" + - " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement'/>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + - " <binding>http://*/deployment/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.BadgeApiHandler'>\n" + - " <binding>http://*/badge/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.CliApiHandler'>\n" + - " <binding>http://*/cli/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>\n" + - " <binding>http://*/controller/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + - " <binding>http://*/os/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" + - " <binding>http://*/zone/v2</binding>\n" + - " <binding>http://*/zone/v2/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.configserver.ConfigServerApiHandler'>\n" + - " <binding>http://*/configserver/v1</binding>\n" + - " <binding>http://*/configserver/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.flags.AuditedFlagsHandler'>\n" + - " <binding>http://*/flags/v1</binding>\n" + - " <binding>http://*/flags/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.user.UserApiHandler'>\n" + - " <binding>http://*/user/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.routing.RoutingApiHandler'>\n" + - " <binding>http://*/routing/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.changemanagement.ChangeManagementApiHandler'>\n" + - " <binding>http://*/changemanagement/v1/*</binding>\n" + - " </handler>\n" + - variablePartXml() + - "</container>"; + return """ + <container version='1.0'> + <config name="container.handler.threadpool"> + <maxthreads>10</maxthreads> + </config> + <config name="cloud.config.configserver"> + <system>%s</system> + </config> + <config name="vespa.hosted.rotation.config.rotations"> + <rotations> + <item key="rotation-id-1">rotation-fqdn-1</item> + <item key="rotation-id-2">rotation-fqdn-2</item> + <item key="rotation-id-3">rotation-fqdn-3</item> + <item key="rotation-id-4">rotation-fqdn-4</item> + <item key="rotation-id-5">rotation-fqdn-5</item> + </rotations> + </config> + + <accesslog type='disabled'/> + + <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/> + <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/> + <component id='com.yahoo.vespa.curator.mock.MockCurator'/> + <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/> + <component id='com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock'/> + <component id='com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock'/> + <component id='com.yahoo.vespa.hosted.controller.Controller'/> + <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/> + <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/> + <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository'/> + <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement'/> + + <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'> + <binding>http://localhost/deployment/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.BadgeApiHandler'> + <binding>http://localhost/badge/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.CliApiHandler'> + <binding>http://localhost/cli/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'> + <binding>http://localhost/controller/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'> + <binding>http://localhost/os/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'> + <binding>http://localhost/zone/v2</binding> + <binding>http://localhost/zone/v2/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.configserver.ConfigServerApiHandler'> + <binding>http://localhost/configserver/v1</binding> + <binding>http://localhost/configserver/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.flags.AuditedFlagsHandler'> + <binding>http://localhost/flags/v1</binding> + <binding>http://localhost/flags/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.user.UserApiHandler'> + <binding>http://localhost/user/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.routing.RoutingApiHandler'> + <binding>http://localhost/routing/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.changemanagement.ChangeManagementApiHandler'> + <binding>http://localhost/changemanagement/v1/*</binding> + </handler> + %s + </container> + """.formatted(system().value(), variablePartXml()); } protected SystemName system() { @@ -123,14 +127,14 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + - " <binding>http://*/application/v4/*</binding>\n" + + " <binding>http://localhost/application/v4/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + - " <binding>http://*/athenz/v1/*</binding>\n" + + " <binding>http://localhost/athenz/v1/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + - " <binding>http://*/zone/v1</binding>\n" + - " <binding>http://*/zone/v1/*</binding>\n" + + " <binding>http://localhost/zone/v1</binding>\n" + + " <binding>http://localhost/zone/v1/*</binding>\n" + " </handler>\n" + " <http>\n" + @@ -140,7 +144,7 @@ public class ControllerContainerTest { " <filter id='com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock'/>\n" + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.AthenzRoleFilter'/>\n" + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + - " <binding>http://*/*</binding>\n" + + " <binding>http://localhost/*</binding>\n" + " </request-chain>\n" + " </filtering>\n" + " </http>\n"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index aa9cb57f541..73d4daf92da 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -85,24 +85,6 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { } @Test - void setting_and_deleting_instrument() { - assertTrue(billingController.getDefaultInstrument(tenant).isEmpty()); - - var instrumentRequest = request("/billing/v1/tenant/tenant1/instrument", PATCH) - .data("{\"active\": \"id-1\"}") - .roles(tenantRole); - - tester.assertResponse(instrumentRequest, "OK"); - assertEquals("id-1", billingController.getDefaultInstrument(tenant).get().getId()); - - var deleteInstrumentRequest = request("/billing/v1/tenant/tenant1/instrument/id-1", DELETE) - .roles(tenantRole); - - tester.assertResponse(deleteInstrumentRequest, "OK"); - assertTrue(billingController.getDefaultInstrument(tenant).isEmpty()); - } - - @Test void response_list_bills() { var bill = createBill(); @@ -197,15 +179,6 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { } @Test - void setting_plans() { - var planRequest = request("/billing/v1/tenant/tenant1/plan", PATCH) - .data("{\"plan\": \"new-plan\"}") - .roles(tenantRole); - tester.assertResponse(planRequest, "Plan: new-plan"); - assertEquals("new-plan", billingController.getPlan(tenant).value()); - } - - @Test void csv_export() { var bill = createBill(); billingController.addBill(tenant, bill, true); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index c62a9f1399f..857dcbac6fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -74,24 +74,16 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { } @Test - void require_admin_for_update_plan() { - var request = request("/billing/v2/tenant/" + tenant.value(), Request.Method.PATCH) - .data("{\"plan\": \"paid\"}"); - - var forbidden = request.roles(tenantReader); - tester.assertResponse(forbidden, ACCESS_DENIED, 403); - var success = request.roles(tenantAdmin); - tester.assertResponse(success, """ - {"tenant":"tenant1","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"collection":"AUTO"}"""); - } - - @Test void require_accountant_for_update_collection() { var request = request("/billing/v2/tenant/" + tenant.value(), Request.Method.PATCH) .data("{\"collection\": \"INVOICE\"}"); var forbidden = request.roles(tenantAdmin); - tester.assertResponse(forbidden, "{\"error-code\":\"FORBIDDEN\",\"message\":\"Only accountant can change billing method\"}", 403); + tester.assertResponse(forbidden, """ + { + "code" : 403, + "message" : "Access denied" + }""", 403); var success = request.roles(financeAdmin); tester.assertResponse(success, """ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java index d0d169720d0..7f826566ebc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/horizon/TsdbQueryRewriterTest.java @@ -3,11 +3,8 @@ package com.yahoo.vespa.hosted.controller.restapi.horizon; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.SlimeUtils; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java index fe59c09fb8f..38019ec725b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.playground; +import ai.vespa.validation.StringWrapper; import com.yahoo.application.Networking; import com.yahoo.component.Version; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -20,13 +23,26 @@ import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Paths; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Comparator; +import java.util.Deque; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Comparator.comparing; +import static java.util.Comparator.naturalOrder; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toSet; public class DeploymentPlayground extends ControllerContainerTest { @@ -64,11 +80,12 @@ public class DeploymentPlayground extends ControllerContainerTest { domainMock.markAsVespaTenant(); domainMock.admin(AllowingFilter.user.getIdentity()); + ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml()); Map<String, DeploymentContext> instances = new LinkedHashMap<>(); - for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100")) - instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name)); + for (var instance : applicationPackage.deploymentSpec().instances()) + instances.put(instance.name().value(), deploymentTester.newDeploymentContext("demo", "app", instance.name().value())); - instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())).deploy(); + instances.values().iterator().next().submit(applicationPackage); repl(instances); } @@ -112,10 +129,13 @@ public class DeploymentPlayground extends ControllerContainerTest { deploymentTester.controllerTester().computeVersionStatus(); break; case "submit": - instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml(), + ValidationId.deploymentRemoval), + command.length == 1 ? 2 : Integer.parseInt(command[1])); break; case "resubmit": - instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml(), + ValidationId.deploymentRemoval)); break; case "advance": deploymentTester.clock().advance(Duration.ofMinutes(Long.parseLong(command[1]))); @@ -130,6 +150,11 @@ public class DeploymentPlayground extends ControllerContainerTest { default: System.err.println("Cannot run '" + String.join(" ", command) + "'"); } + Set<String> names = instances.values().iterator().next().application().deploymentSpec().instanceNames().stream().map(StringWrapper::value).collect(toSet()); + instances.keySet().removeIf(not(names::contains)); + names.removeIf(instances.keySet()::contains); + for (String name : names) + instances.put(name, deploymentTester.newDeploymentContext("demo", "app", name)); } } catch (Throwable t) { @@ -142,21 +167,35 @@ public class DeploymentPlayground extends ControllerContainerTest { while ( ! Thread.currentThread().isInterrupted()) { try { synchronized (monitor) { - monitor.wait(6000); + monitor.wait(1000); + if ( ! on.get()) + continue; + } + + deploymentTester.clock().advance(Duration.ofSeconds(60)); + deploymentTester.runner().run(); + deploymentTester.triggerJobs(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.upgrader().run(); + + synchronized (monitor) { + monitor.wait(1000); if ( ! on.get()) continue; + } + deploymentTester.clock().advance(Duration.ofSeconds(60)); - System.err.println("auto running"); - deploymentTester.clock().advance(Duration.ofSeconds(60)); - deploymentTester.controllerTester().computeVersionStatus(); - deploymentTester.outstandingChangeDeployer().run(); - deploymentTester.upgrader().run(); - deploymentTester.triggerJobs(); - deploymentTester.runner().run(); - for (Run run : deploymentTester.jobs().active()) - if (run.versions().sourcePlatform().map(run.versions().targetPlatform()::equals).orElse(true) || Math.random() < 0.4) - instances.get(run.id().application().instance().value()).runJob(run.id().type()); + List<Run> active = deploymentTester.jobs().active(); + if ( ! active.isEmpty()) { + Run run = active.stream() + .min(comparing(current -> deploymentTester.jobs().last(current.id().job()).map(Run::start) + .orElse(Instant.EPOCH))).get(); + if (run.versions().sourcePlatform().map(run.versions().targetPlatform()::equals).orElse(true) || Math.random() < 0.4) { + instances.get(run.id().application().instance().value()).runJob(run.id().type()); + } } + } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -167,6 +206,58 @@ public class DeploymentPlayground extends ControllerContainerTest { } } + /** + void auto(Map<String, DeploymentContext> instances, AtomicBoolean on) { + BooleanSupplier runJob = () -> { + List<Run> active = deploymentTester.jobs().active(); + if ( ! active.isEmpty()) { + Run run = active.stream() + .min(comparing(current -> deploymentTester.jobs().last(current.id().job()).map(Run::start) + .orElse(Instant.EPOCH))).get(); + if (run.versions().sourcePlatform().map(run.versions().targetPlatform()::equals).orElse(true) || Math.random() < 0.4) { + instances.get(run.id().application().instance().value()).runJob(run.id().type()); + return false; + } + } + return true; + }; + List<Runnable> defaultTasks = List.of(() -> deploymentTester.runner().run(), + () -> deploymentTester.triggerJobs(), + () -> deploymentTester.outstandingChangeDeployer().run(), + () -> deploymentTester.upgrader().run(), + () -> deploymentTester.controllerTester().computeVersionStatus()); + while ( ! Thread.currentThread().isInterrupted()) { + Deque<BooleanSupplier> tasks = new ArrayDeque<>(); + tasks.push(runJob); + for (Runnable task : defaultTasks) + tasks.push(() -> { + task.run(); + return true; + }); + + while ( ! tasks.isEmpty()) + try { + synchronized (monitor) { + monitor.wait(1000); + if ( ! on.get()) + break; + + deploymentTester.clock().advance(Duration.ofSeconds(60)); + BooleanSupplier task = tasks.pop(); + if ( ! task.getAsBoolean()) + tasks.push(task); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + */ + void run(DeploymentContext instance, BiConsumer<DeploymentContext, JobType> action, List<String> jobs) { Set<JobId> haveRun = new HashSet<>(); boolean triggered = true; @@ -186,49 +277,50 @@ public class DeploymentPlayground extends ControllerContainerTest { @Override protected String variablePartXml() { - return " <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" + - - " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + - " <binding>http://*/application/v4/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + - " <binding>http://*/athenz/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + - " <binding>http://*/zone/v1</binding>\n" + - " <binding>http://*/zone/v1/*</binding>\n" + - " </handler>\n" + - - " <http>\n" + - " <server id='default' port='8080' />\n" + - " <filtering>\n" + - " <request-chain id='default'>\n" + - " <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsPreflightRequestFilter'>\n" + - " <config name=\"jdisc.http.filter.security.cors.cors-filter\">" + - " <allowedUrls>\n" + - " <item>http://localhost:3000</item>\n" + - " <item>http://localhost:8080</item>\n" + - " </allowedUrls>\n" + - " </config>\n" + - " </filter>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.restapi.playground.AllowingFilter'/>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " <response-chain id='responses'>\n" + - " <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsResponseFilter'>\n" + - " <config name=\"jdisc.http.filter.security.cors.cors-filter\">" + - " <allowedUrls>\n" + - " <item>http://localhost:3000</item>\n" + - " <item>http://localhost:8080</item>\n" + - " </allowedUrls>\n" + - " </config>\n" + - " </filter>\n" + - " <binding>http://*/*</binding>\n" + - " </response-chain>\n" + - " </filtering>\n" + - " </http>\n"; + return """ + <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/> + <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/> + + <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'> + <binding>http://localhost/application/v4/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'> + <binding>http://localhost/athenz/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'> + <binding>http://localhost/zone/v1</binding> + <binding>http://localhost/zone/v1/*</binding> + </handler> + + <http> + <server id='default' port='8080' /> + <filtering> + <request-chain id='default'> + <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsPreflightRequestFilter'> + <config name="jdisc.http.filter.security.cors.cors-filter"> + <allowedUrls> + <item>http://localhost:3000</item> + <item>http://localhost:8080</item> + </allowedUrls> + </config> + </filter> + <filter id='com.yahoo.vespa.hosted.controller.restapi.playground.AllowingFilter'/> + <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/> + <binding>http://localhost/*</binding> + </request-chain> + <response-chain id='responses'> + <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsResponseFilter'> + <config name="jdisc.http.filter.security.cors.cors-filter"> <allowedUrls> + <item>http://localhost:3000</item> + <item>http://localhost:8080</item> + </allowedUrls> + </config> + </filter> + <binding>http://localhost/*</binding> + </response-chain> + </filtering> + </http> + """; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml index cc9feb90d90..61749cd89f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml @@ -1,83 +1,132 @@ -<deployment> - <instance id='alpha'> - <upgrade rollout="simultaneous" revision-change="when-failing" revision-target="next" /> - <test /> - <staging /> - </instance> - <instance id='beta'> - <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="latest" /> - <prod> - <region>us-east-3</region> - <test>us-east-3</test> - </prod> - </instance> - <instance id='prod5'> - <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" /> - <prod> +<deployment version='1.0'> + <notifications> + <email role="author" /> + </notifications> + + <parallel> + <instance id="omega"> <!-- Eats extra system tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <steps> <parallel> - <steps> + <instance id="alpha-1"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-2"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-3"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + </parallel> + + <instance id="alpha-4"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> <region>us-east-3</region> <test>us-east-3</test> - </steps> - <steps> - <region>us-central-1</region> - <test>us-central-1</test> - </steps> - <steps> <region>us-west-1</region> + </prod> + </instance> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> <test>us-west-1</test> - </steps> + </prod> + </instance> + + <parallel> + <instance id="prod5"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> </parallel> - </prod> - </instance> - <instance id='prod25'> - <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" /> - <prod> + <parallel> - <steps> - <region>us-east-3</region> - <test>us-east-3</test> - </steps> - <steps> - <region>us-central-1</region> - <test>us-central-1</test> - </steps> - <steps> - <region>us-west-1</region> - <test>us-west-1</test> - </steps> + <instance id="prod15"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-central-1</region> + <region>eu-west-1</region> + <region>aws-us-east-1a</region> + </parallel> + <delay hours='8' /> + <test>us-central-1</test> + </prod> + </instance> + + <instance id="prod25"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-west-1</region> + <region>us-east-3</region> + <region>ap-northeast-1</region> + </parallel> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> </parallel> - </prod> - </instance> - <instance id='prod100'> - <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" /> - <prod> - <steps> - <parallel> - <steps> - <region>eu-west-1</region> - <test>eu-west-1</test> - </steps> - <steps> - <region>ap-northeast-1</region> - <test>ap-northeast-1</test> - </steps> - </parallel> + <parallel> - <steps> - <region>us-east-3</region> - <test>us-east-3</test> - </steps> - <steps> - <region>us-central-1</region> - <test>us-central-1</test> - </steps> - <steps> - <region>us-west-1</region> - <test>us-west-1</test> - </steps> + <instance id="prod50"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-central-1</region> + <region>eu-west-1</region> + <region>aws-us-east-1a</region> + </parallel> + </prod> + </instance> + + <instance id="prod100"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-west-1</region> + <region>us-east-3</region> + <region>ap-northeast-1</region> + </parallel> + </prod> + </instance> </parallel> - </steps> - </prod> - </instance> -</deployment> + + </steps> + </parallel> + +</deployment>
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_alt_full.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_alt_full.xml new file mode 100644 index 00000000000..61749cd89f0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_alt_full.xml @@ -0,0 +1,132 @@ +<deployment version='1.0'> + <notifications> + <email role="author" /> + </notifications> + + <parallel> + <instance id="omega"> <!-- Eats extra system tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <steps> + <parallel> + <instance id="alpha-1"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-2"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-3"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + </parallel> + + <instance id="alpha-4"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + <region>us-west-1</region> + </prod> + </instance> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <parallel> + <instance id="prod5"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> + </parallel> + + <parallel> + <instance id="prod15"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-central-1</region> + <region>eu-west-1</region> + <region>aws-us-east-1a</region> + </parallel> + <delay hours='8' /> + <test>us-central-1</test> + </prod> + </instance> + + <instance id="prod25"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-west-1</region> + <region>us-east-3</region> + <region>ap-northeast-1</region> + </parallel> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> + </parallel> + + <parallel> + <instance id="prod50"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-central-1</region> + <region>eu-west-1</region> + <region>aws-us-east-1a</region> + </parallel> + </prod> + </instance> + + <instance id="prod100"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-west-1</region> + <region>us-east-3</region> + <region>ap-northeast-1</region> + </parallel> + </prod> + </instance> + </parallel> + + </steps> + </parallel> + +</deployment>
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_alternative.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_alternative.xml new file mode 100644 index 00000000000..8bd24e977b1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_alternative.xml @@ -0,0 +1,67 @@ +<deployment version='1.0'> + <notifications> + <email role="author" /> + </notifications> + + <parallel> + <instance id="omega"> <!-- Eats extra system tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <steps> + <parallel> + <instance id="alpha-1"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-2"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-3"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + </parallel> + + <instance id="alpha-4"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + <region>us-west-1</region> + </prod> + </instance> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + </prod> + </instance> + </steps> + </parallel> + +</deployment> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_base.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_base.xml new file mode 100644 index 00000000000..e3d7933d5e7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_base.xml @@ -0,0 +1,63 @@ +<deployment version='1.0'> + <notifications> + <email role="author" /> + </notifications> + + <parallel> + <instance id="omega"> <!-- Eats extra system tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <steps> + <parallel> + <instance id="alpha-1"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-2"> <!-- Runs one third of the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-3"> <!-- Runs one third of the system tests, and the stress tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + </prod> + </instance> + </parallel> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + </prod> + </instance> + </steps> + </parallel> + +</deployment> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_full.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_full.xml new file mode 100644 index 00000000000..5a7b44f966d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_full.xml @@ -0,0 +1,127 @@ +<deployment version='1.0'> + <notifications> + <email role="author" /> + </notifications> + + <parallel> + <instance id="omega"> <!-- Eats extra system and staging tests --> + <upgrade policy='canary' revision-target='latest' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + </instance> + + <steps> + <parallel> + <instance id="alpha-1"> <!-- Runs half the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-2"> <!-- Runs half the system and staging tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="alpha-3"> <!-- Runs one third of the system tests, and the stress tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + </prod> + </instance> + </parallel> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <parallel> + <instance id="prod5"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> + </parallel> + + <parallel> + <instance id="prod15"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-central-1</region> + <region>eu-west-1</region> + <region>aws-us-east-1a</region> + </parallel> + <delay hours='8' /> + <test>us-central-1</test> + </prod> + </instance> + + <instance id="prod25"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-west-1</region> + <region>us-east-3</region> + <region>ap-northeast-1</region> + </parallel> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> + </parallel> + + <parallel> + <instance id="prod50"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-central-1</region> + <region>eu-west-1</region> + <region>aws-us-east-1a</region> + </parallel> + </prod> + </instance> + + <instance id="prod100"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <parallel> + <region>us-west-1</region> + <region>us-east-3</region> + <region>ap-northeast-1</region> + </parallel> + </prod> + </instance> + </parallel> + + </steps> + </parallel> + +</deployment> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simple.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simple.xml new file mode 100644 index 00000000000..cdcaadbd957 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simple.xml @@ -0,0 +1,44 @@ +<deployment version='1.0'> + + <parallel> + <instance id="omega"> <!-- Eats extra system tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <steps> + <instance id="alpha"> <!-- Runs system and stress tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> <!-- Not really a production deployment, but used in stress tests --> + </prod> + </instance> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + </prod> + </instance> + </steps> + </parallel> + +</deployment> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simpler.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simpler.xml new file mode 100644 index 00000000000..f0b0ae79d81 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simpler.xml @@ -0,0 +1,30 @@ +<deployment version='1.0'> + + <instance id="alpha"> <!-- Runs system and stress tests --> + <upgrade policy='conservative' revision-target='next' revision-change='when-failing' rollout='separate' /> + <test /> + <staging /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + </instance> + + <instance id="beta"> + <!-- Consider allowing risk based rollout with when-failing ... --> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='3' max-risk='12' max-idle-hours='8' /> + <!--block-change revision="false" days="sun,mon,tue,thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' min-risk='6' max-risk='12' max-idle-hours='32' /> + <!--block-change version="false" days="thu,fri,sat" hours="0-23" time-zone="UTC"/--> + <!--block-change revision="false" days="sun,mon,tue,wed,fri,sat" hours="0-23" time-zone="UTC"/--> + <prod> + <region>us-west-1</region> + </prod> + </instance> + +</deployment> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simplest.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simplest.xml new file mode 100644 index 00000000000..b023587b6a9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment_simplest.xml @@ -0,0 +1,39 @@ +<deployment version='1.0'> + + <instance id="beta"> <!-- Runs system and production tests --> + <test /> + <staging /> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' /> + <prod> + <region>us-west-1</region> + <delay hours='1' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod5"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' /> + <prod> + <region>us-west-1</region> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod25"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' /> + <prod> + <region>us-west-1</region> + <delay hours='8' /> + <test>us-west-1</test> + </prod> + </instance> + + <instance id="prod100"> + <upgrade policy='conservative' revision-target='next' revision-change='when-clear' rollout='separate' /> + <prod> + <region>us-west-1</region> + </prod> + </instance> + +</deployment> diff --git a/dist/vespa.spec b/dist/vespa.spec index 82bd4d0917c..b6882f6d4d5 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -778,6 +778,7 @@ fi %{_prefix}/libexec %exclude %{_prefix}/libexec/vespa_ann_benchmark %exclude %{_prefix}/libexec/vespa/common-env.sh +%exclude %{_prefix}/libexec/vespa/script-utils %exclude %{_prefix}/libexec/vespa/find-pid %exclude %{_prefix}/libexec/vespa/node-admin.sh %exclude %{_prefix}/libexec/vespa/standalone-container.sh @@ -842,6 +843,7 @@ fi %dir %{_prefix}/libexec %dir %{_prefix}/libexec/vespa %{_prefix}/libexec/vespa/common-env.sh +%{_prefix}/libexec/vespa/script-utils %{_prefix}/libexec/vespa/find-pid %{_prefix}/libexec/vespa/vespa-curl-wrapper diff --git a/document/src/tests/annotation/annotation_test.cpp b/document/src/tests/annotation/annotation_test.cpp index 2700cfcf96f..b18fa113d9b 100644 --- a/document/src/tests/annotation/annotation_test.cpp +++ b/document/src/tests/annotation/annotation_test.cpp @@ -156,8 +156,8 @@ TEST("requireThatAnnotationsCanHaveValues") { TEST("requireThatAnnotationsCanReferenceAnnotations") { auto root = std::make_unique<SpanList>(); SpanTree tree("html", std::move(root)); - size_t san_index = tree.annotate(makeUP(new Annotation(text_type))); - size_t fran_index = tree.annotate(makeUP(new Annotation(text_type))); + size_t san_index = tree.annotate(Annotation(text_type)); + size_t fran_index = tree.annotate(Annotation(text_type)); AnnotationReferenceDataType annotation_ref_type(text_type, 101); ArrayDataType array_type(annotation_ref_type); diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 40d78327ab4..7528878bfb5 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -626,7 +626,7 @@ TEST("requireThatDocumentWithDocumentCanBeSerialized") { const AnnotationType *a_type =my_repo.getAnnotationType(*inner_type, a_id); StringFieldValue str("foo"); auto tree = std::make_unique<SpanTree>("name", std::make_unique<Span>(0, 3)); - tree->annotate(std::make_unique<Annotation>(*a_type)); + tree->annotate(Annotation(*a_type)); setSpanTree(str, *tree); diff --git a/document/src/vespa/document/annotation/spantree.cpp b/document/src/vespa/document/annotation/spantree.cpp index d483d6c08ca..4d24b4ce788 100644 --- a/document/src/vespa/document/annotation/spantree.cpp +++ b/document/src/vespa/document/annotation/spantree.cpp @@ -12,20 +12,20 @@ namespace document { SpanTree::~SpanTree() = default; size_t -SpanTree::annotate(std::unique_ptr<Annotation> annotation_) { - _annotations.push_back(std::move(*annotation_)); +SpanTree::annotate(Annotation&& annotation_) { + _annotations.push_back(std::move(annotation_)); return _annotations.size() - 1; } size_t -SpanTree::annotate(const SpanNode &node, std::unique_ptr<Annotation> annotation_) { - annotation_->setSpanNode(node); +SpanTree::annotate(const SpanNode &node, Annotation&& annotation_) { + annotation_.setSpanNode(node); return annotate(std::move(annotation_)); } size_t -SpanTree::annotate(const SpanNode &node, const AnnotationType &type) { - return annotate(node, std::make_unique<Annotation>(type)); +SpanTree::annotate(const SpanNode &node, const AnnotationType &annotation_type) { + return annotate(node, Annotation(annotation_type)); } void diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h index 7635350025b..03ee820466c 100644 --- a/document/src/vespa/document/annotation/spantree.h +++ b/document/src/vespa/document/annotation/spantree.h @@ -29,9 +29,9 @@ public: ~SpanTree(); // The annotate functions return the annotation index. - size_t annotate(std::unique_ptr<Annotation> annotation); - size_t annotate(const SpanNode &node, std::unique_ptr<Annotation> a); - size_t annotate(const SpanNode &node, const AnnotationType &a_type); + size_t annotate(Annotation&& annotation_); + size_t annotate(const SpanNode& node, Annotation&& annotation_); + size_t annotate(const SpanNode& node, const AnnotationType& annotation_type); Annotation & annotation(size_t index) { return _annotations[index]; } const Annotation & annotation(size_t index) const { return _annotations[index]; } diff --git a/documentapi/src/Doxyfile b/documentapi/src/Doxyfile deleted file mode 100644 index e0a9714d2e9..00000000000 --- a/documentapi/src/Doxyfile +++ /dev/null @@ -1,1213 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.4.1 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = documentapi - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../../doc - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, -# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, -# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, -# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, -# Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. - -SHOW_DIRECTORIES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from the -# version control system). Doxygen will invoke the program by executing (via -# popen()) the command <command> <input-file>, where <command> is the value of -# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file -# provided by doxygen. Whatever the progam writes to standard output -# is used as the file version. See the manual for examples. - -FILE_VERSION_FILTER = - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be abled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. - -WARN_NO_PARAMDOC = NO - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = documentapi - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx -# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm - -FILE_PATTERNS = *.h *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or -# directories that are symbolic links (a Unix filesystem feature) are excluded -# from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = IAM_DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse -# the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base -# or super classes. Setting the tag to NO turns the diagrams off. Note that -# this option is superseded by the HAVE_DOT option below. This is only a -# fallback. It is recommended to install and use dot, since it yields more -# powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for groups, showing the direct groups dependencies - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES -# then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include -# relations between the files in the directories. - -DIRECTORY_GRAPH = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes -# that lay further from the root node will be omitted. Note that setting this -# option to 1 or 2 may greatly reduce the computation time needed for large -# code bases. Also note that a graph may be further truncated if the graph's -# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH -# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), -# the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, which results in a white background. -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). - -DOT_TRANSPARENT = NO - -# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) -# support this, this feature is disabled by default. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 8c3ea7b1eed..a3c1ea7ae41 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -340,7 +340,7 @@ FBench::Main(int argc, char *argv[]) { // parameters with default values. int numClients = 10; - int cycleTime = 1000; + int cycleTime = 0; int byteLimit = 0; int ignoreCount = 0; int seconds = 60; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5527a565cf4..90830b04124 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -70,6 +70,13 @@ public class Flags { "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag PHRASE_OPTIMIZATION = defineStringFlag( + "phrase-optimization", "split", + List.of("baldersheim"), "2022-08-28", "2023-01-01", + "Select phase optimization, valid values are 'split', 'off'.", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "THROUGHPUT", List.of("baldersheim"), "2020-12-02", "2023-01-01", @@ -127,7 +134,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( - "use-three-phase-updates", false, + "use-three-phase-updates", true, List.of("vekterli"), "2020-12-02", "2022-10-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", @@ -248,7 +255,7 @@ public class Flags { public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, - List.of("bjorncs", "tokle"), "2021-02-10", "2022-09-01", + List.of("bjorncs", "tokle"), "2021-02-10", "2022-11-01", "Allowed Athenz proxy identities", "takes effect at redeployment"); @@ -276,7 +283,7 @@ public class Flags { public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, - List.of("geirst", "vekterli"), "2021-07-16", "2022-09-01", + List.of("geirst", "vekterli"), "2021-07-16", "2022-11-01", "Minimum ratio of nodes that have to be available (i.e. not Down) in any hierarchic content cluster group for the group to be Up", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -305,7 +312,7 @@ public class Flags { public static final UnboundBooleanFlag UNORDERED_MERGE_CHAINING = defineFeatureFlag( "unordered-merge-chaining", true, - List.of("vekterli", "geirst"), "2021-11-15", "2022-09-01", + List.of("vekterli", "geirst"), "2021-11-15", "2022-11-01", "Enables the use of unordered merge chains for data merge operations", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); @@ -332,13 +339,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_SERVER_OCSP_STAPLING = defineFeatureFlag( - "enable-server-ocsp-stapling", false, - List.of("bjorncs"), "2021-12-17", "2022-09-01", - "Enable server OCSP stapling for jdisc containers", - "Takes effect on redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_DATA_HIGHWAY_IN_AWS = defineFeatureFlag( "enable-data-highway-in-aws", false, List.of("hmusum"), "2022-01-06", "2022-10-01", @@ -372,7 +372,7 @@ public class Flags { public static final UnboundIntFlag PERSISTENCE_THROTTLING_WINDOW_SIZE = defineIntFlag( "persistence-throttling-window-size", -1, - List.of("vekterli"), "2022-02-23", "2022-09-01", + List.of("vekterli"), "2022-02-23", "2022-11-01", "If greater than zero, sets both min and max window size to the given number, effectively " + "turning dynamic throttling into a static throttling policy. " + "Only applies if DYNAMIC policy is used.", @@ -381,14 +381,14 @@ public class Flags { public static final UnboundDoubleFlag PERSISTENCE_THROTTLING_WS_RESIZE_RATE = defineDoubleFlag( "persistence-throttling-ws-resize-rate", 3.0, - List.of("vekterli"), "2022-02-23", "2022-09-01", + List.of("vekterli"), "2022-02-23", "2022-11-01", "Sets the dynamic throttle policy resize rate. Only applies if DYNAMIC policy is used.", "Takes effect on redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag PERSISTENCE_THROTTLING_OF_MERGE_FEED_OPS = defineFeatureFlag( "persistence-throttling-of-merge-feed-ops", true, - List.of("vekterli"), "2022-02-24", "2022-09-01", + List.of("vekterli"), "2022-02-24", "2022-11-01", "If true, each put/remove contained within a merge is individually throttled as if it " + "were a put/remove from a client. If false, merges are throttled at a persistence thread " + "level, i.e. per ApplyBucketDiff message, regardless of how many document operations " + @@ -433,7 +433,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, - List.of("tokle"), "2022-05-09", "2022-09-01", + List.of("tokle"), "2022-05-09", "2022-11-01", "Enable or disable proxy protocol mixed mode", "Takes effect on redeployment", APPLICATION_ID); @@ -454,7 +454,7 @@ public class Flags { public static final UnboundBooleanFlag USE_YUM_PROXY_V2 = defineFeatureFlag( "use-yumproxy-v2", false, - List.of("tokle"), "2022-05-05", "2022-09-01", + List.of("tokle"), "2022-05-05", "2022-11-01", "Use yumproxy-v2", "Takes effect on host admin restart", HOSTNAME); @@ -468,7 +468,7 @@ public class Flags { public static final UnboundBooleanFlag SEPARATE_METRIC_CHECK_CONFIG = defineFeatureFlag( "separate-metric-check-config", false, - List.of("olaa"), "2022-07-04", "2022-09-01", + List.of("olaa"), "2022-07-04", "2022-11-01", "Determines whether one metrics config check should be written per Vespa node", "Takes effect on next tick", HOSTNAME); diff --git a/fnet/src/Doxyfile b/fnet/src/Doxyfile deleted file mode 100644 index 8e0f0232f35..00000000000 --- a/fnet/src/Doxyfile +++ /dev/null @@ -1,939 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.2.15 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# General configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = FNET - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = head - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../doc/doxygen - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Chinese, Croatian, Czech, Danish, Dutch, Finnish, French, -# German, Greek, Hungarian, Italian, Japanese, Korean, Norwegian, Polish, -# Portuguese, Romanian, Russian, Slovak, Slovene, Spanish and Swedish. - -OUTPUT_LANGUAGE = English - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these class will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower case letters. If set to YES upper case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are adviced to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explict @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# reimplements. - -INHERIT_DOCS = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consist of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = fnet fnet/frt fnet/testkit - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl - -FILE_PATTERNS = *.h *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = NO - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse. - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = FNET_ FRT_ - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the Html help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript and frames is required (for instance Mozilla, Netscape 4.0+, -# or Internet explorer 4.0+). Note that for large projects the tree generation -# can take a very long time. In such cases it is better to disable this feature. -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimised for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assigments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_XML = NO - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = IAM_DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_PREDEF_ONLY tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line and do not end with a semicolon. Such function macros are typically -# used for boiler-plate code, and will confuse the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES tag can be used to specify one or more tagfiles. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in Html, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superceded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yield more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = YES - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermedate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO - -# The CGI_NAME tag should be the name of the CGI script that -# starts the search engine (doxysearch) with the correct parameters. -# A script with this name will be generated by doxygen. - -CGI_NAME = search.cgi - -# The CGI_URL tag should be the absolute URL to the directory where the -# cgi binaries are located. See the documentation of your http daemon for -# details. - -CGI_URL = - -# The DOC_URL tag should be the absolute URL to the directory where the -# documentation is located. If left blank the absolute path to the -# documentation, with file:// prepended to it, will be used. - -DOC_URL = - -# The DOC_ABSPATH tag should be the absolute path to the directory where the -# documentation is located. If left blank the directory on the local machine -# will be used. - -DOC_ABSPATH = - -# The BIN_ABSPATH tag must point to the directory where the doxysearch binary -# is installed. - -BIN_ABSPATH = /usr/local/bin/ - -# The EXT_DOC_PATHS tag can be used to specify one or more paths to -# documentation generated for other projects. This allows doxysearch to search -# the documentation for these projects as well. - -EXT_DOC_PATHS = diff --git a/fnet/src/vespa/fnet/frt/require_capabilities.cpp b/fnet/src/vespa/fnet/frt/require_capabilities.cpp index 5f87f98436e..6996557c91e 100644 --- a/fnet/src/vespa/fnet/frt/require_capabilities.cpp +++ b/fnet/src/vespa/fnet/frt/require_capabilities.cpp @@ -29,7 +29,7 @@ FRT_RequireCapabilities::allow(FRT_RPCRequest& req) const noexcept "Peer at %s with %s. Call requires %s, but peer has %s", ((mode == CapabilityEnforcementMode::LogOnly) ? "(Dry-run only, not enforced): " : ""), method_name.c_str(), peer_spec.c_str(), - to_string(auth_ctx.peer_credentials()).c_str(), + auth_ctx.peer_credentials().to_string().c_str(), _required_capabilities.to_string().c_str(), auth_ctx.capabilities().to_string().c_str()); return (mode != CapabilityEnforcementMode::Enforce); diff --git a/fsa/doc/Doxyfile b/fsa/doc/Doxyfile deleted file mode 100644 index cc4d0ab88c4..00000000000 --- a/fsa/doc/Doxyfile +++ /dev/null @@ -1,1099 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.3.5 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = fsa - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = 2.0.1 - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, -# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en -# (Japanese with English messages), Korean, Norwegian, Polish, Portuguese, -# Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is used -# as the annotated text. Otherwise, the brief description is used as-is. If left -# blank, the following values are used ("$name" is automatically replaced with the -# name of the entity): "The $name class" "The $name widget" "The $name file" -# "is" "provides" "specifies" "contains" "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = ../src - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl *.cs *.php *.php3 *.inc - -FILE_PATTERNS = *.h *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = YES - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 3 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = NO - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superseded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = YES - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes that -# lay further from the root node will be omitted. Note that setting this option to -# 1 or 2 may greatly reduce the computation time needed for large code bases. Also -# note that a graph may be further truncated if the graph's image dimensions are -# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). -# If 0 is used for the depth value (the default), the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/jdisc-security-filters/src/main/resources/configdefinitions/jdisc.http.filter.security.rule.rule-based-filter.def b/jdisc-security-filters/src/main/resources/configdefinitions/jdisc.http.filter.security.rule.rule-based-filter.def deleted file mode 100644 index d619f5ff735..00000000000 --- a/jdisc-security-filters/src/main/resources/configdefinitions/jdisc.http.filter.security.rule.rule-based-filter.def +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: remove this def when oldest hosted model no longer uses it. - -namespace=jdisc.http.filter.security.rule - -dryrun bool default=false -defaultRule.action enum { ALLOW, BLOCK } -defaultRule.blockResponseCode int default=403 -defaultRule.blockResponseMessage string default="" -defaultRule.blockResponseHeaders[].name string -defaultRule.blockResponseHeaders[].value string -rule[].name string -rule[].action enum { ALLOW, BLOCK } -rule[].hostNames[] string -rule[].methods[] enum { GET, POST, PUT, PATCH, DELETE } -rule[].pathExpressions[] string -rule[].blockResponseCode int default=403 -rule[].blockResponseMessage string default="" -rule[].blockResponseHeaders[].name string -rule[].blockResponseHeaders[].value string diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java index 6b57b1e90e7..2f90b4e067f 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java @@ -31,7 +31,7 @@ import java.util.stream.Collectors; * @author gjoranv */ public class BundleCollisionHook implements CollisionHook, EventHook, FindHook { - private static Logger log = Logger.getLogger(BundleCollisionHook.class.getName()); + private static final Logger log = Logger.getLogger(BundleCollisionHook.class.getName()); private ServiceRegistration<?> registration; private Map<Bundle, BsnVersion> allowedDuplicates = new HashMap<>(5); @@ -90,7 +90,7 @@ public class BundleCollisionHook implements CollisionHook, EventHook, FindHook { /** * Filters out the set of bundles that should not be visible to the bundle associated with the given context. * If the given context represents one of the allowed duplicates, this method filters out all bundles - * that are duplicates of the allowed duplicates. Otherwise this method filters out the allowed duplicates, + * that are duplicates of the allowed duplicates. Otherwise, this method filters out the allowed duplicates, * so they are not visible to other bundles. */ @Override diff --git a/messagebus/src/Doxyfile b/messagebus/src/Doxyfile deleted file mode 100644 index 36feecfb9e3..00000000000 --- a/messagebus/src/Doxyfile +++ /dev/null @@ -1,1257 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.4.7 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = messagebus - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../../doc - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, -# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, -# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, -# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, -# Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce -# a new page for each member. If set to NO, the documentation of a member will -# be part of the file/class/namespace that contains it. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to -# include (a tag file for) the STL sources as input, then you should -# set this tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. -# func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. - -BUILTIN_STL_SUPPORT = NO - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. The default is NO. - -SHOW_DIRECTORIES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from the -# version control system). Doxygen will invoke the program by executing (via -# popen()) the command <command> <input-file>, where <command> is the value of -# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file -# provided by doxygen. Whatever the program writes to standard output -# is used as the file version. See the manual for examples. - -FILE_VERSION_FILTER = - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be abled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. - -WARN_NO_PARAMDOC = NO - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = messagebus - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx -# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py - -FILE_PATTERNS = *.h \ - *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = messagebus/testlib \ - messagebus/sanity.cpp \ - messagebus/config-messagebus.h \ - messagebus/config-messagebus.cpp - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or -# directories that are symbolic links (a Unix filesystem feature) are excluded -# from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. Note that the wildcards are matched -# against the file with absolute path, so to exclude all test directories -# for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. Otherwise they will link to the documentstion. - -REFERENCES_LINK_SOURCE = YES - -# If the USE_HTAGS tag is set to YES then the references to source code -# will point to the HTML generated by the htags(1) tool instead of doxygen -# built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You -# will need version 4.8.6 or higher. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_DEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = IAM_DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse -# the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base -# or super classes. Setting the tag to NO turns the diagrams off. Note that -# this option is superseded by the HAVE_DOT option below. This is only a -# fallback. It is recommended to install and use dot, since it yields more -# powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for groups, showing the direct groups dependencies - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a caller dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable caller graphs for selected -# functions only using the \callergraph command. - -CALLER_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES -# then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include -# relations between the files in the directories. - -DIRECTORY_GRAPH = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes -# that lay further from the root node will be omitted. Note that setting this -# option to 1 or 2 may greatly reduce the computation time needed for large -# code bases. Also note that a graph may be further truncated if the graph's -# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH -# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), -# the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, which results in a white background. -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). - -DOT_TRANSPARENT = NO - -# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) -# support this, this feature is disabled by default. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 19d3b5b3e43..f7f8a79bec3 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -17,6 +17,7 @@ import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.protect.Process; import com.yahoo.text.Utf8Array; import com.yahoo.text.Utf8String; +import com.yahoo.vespa.defaults.Defaults; import java.util.HashMap; import java.util.List; @@ -151,6 +152,8 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, net.attach(this); if ( ! net.net().waitUntilReady(120)) { Process.dumpThreads(); + String fn = "var/crash/java_pid." + ProcessHandle.current().pid() + ".hprof"; + Process.dumpHeap(Defaults.getDefaults().underVespaHome(fn), true); throw new IllegalStateException("Network failed to become ready in time."); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java index 2908cf39fc8..87dd42d8008 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.google.common.net.InetAddresses; -import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; import java.net.InetAddress; @@ -45,7 +44,7 @@ public class Acl { this(trustedPorts, trustedNodes, Set.of()); } - public List<String> toRules(IPVersion ipVersion, NodeType nodeType) { + public List<String> toRules(IPVersion ipVersion) { List<String> rules = new LinkedList<>(); // We reject with rules instead of using policies @@ -67,26 +66,20 @@ public class Acl { rules.add("-A INPUT -p tcp -m multiport --dports " + joinPorts(trustedPorts) + " -j ACCEPT"); } - // Trust ZooKeeper from other config servers/controllers only - if (nodeType.isConfigServerLike()) { - Set<Integer> zooKeeperPorts = Set.of(2181, 2182, 2183); - List<String> clusterAddresses = getTrustedNodes(ipVersion).stream() - .filter(node -> node.type() == nodeType) - .map(Node::inetAddressString) - .sorted() - .toList(); - for (var ipAddress : clusterAddresses) { - rules.add("-A INPUT -s " + ipAddress + ipVersion.singleHostCidr() + " -p tcp -m multiport --dports " + - joinPorts(zooKeeperPorts) + " -j ACCEPT"); - } - // Reject any other connections to ZooKeeper - rules.add("-A INPUT -p tcp -m multiport --dports " + joinPorts(zooKeeperPorts) + - " -j REJECT --reject-with " + ipVersion.icmpPortUnreachable()); - } - - // Allow traffic from trusted nodes + // Allow traffic from trusted nodes, limited to specific ports, if any getTrustedNodes(ipVersion).stream() - .map(node -> "-A INPUT -s " + node.inetAddressString() + ipVersion.singleHostCidr() + " -j ACCEPT") + .map(node -> { + StringBuilder rule = new StringBuilder(); + rule.append("-A INPUT -s ") + .append(node.inetAddressString()) + .append(ipVersion.singleHostCidr()); + if (!node.ports.isEmpty()) { + rule.append(" -p tcp -m multiport --dports ") + .append(joinPorts(node.ports())); + } + rule.append(" -j ACCEPT"); + return rule.toString(); + }) .sorted() .forEach(rules::add); @@ -103,7 +96,7 @@ public class Acl { } private static String joinPorts(Collection<Integer> ports) { - return ports.stream().map(String::valueOf).sorted().collect(Collectors.joining(",")); + return ports.stream().sorted().map(String::valueOf).collect(Collectors.joining(",")); } public Set<Node> getTrustedNodes() { @@ -158,10 +151,10 @@ public class Acl { return Optional.ofNullable(set).map(Set::copyOf).orElseGet(Set::of); } - public record Node(String hostname, NodeType type, InetAddress inetAddress) { + public record Node(String hostname, InetAddress inetAddress, Set<Integer> ports) { - public Node(String hostname, NodeType type, String ipAddress) { - this(hostname, type, InetAddresses.forString(ipAddress)); + public Node(String hostname, String ipAddress, Set<Integer> ports) { + this(hostname, InetAddresses.forString(ipAddress), ports); } public String inetAddressString() { @@ -173,7 +166,7 @@ public class Acl { return "Node{" + "hostname='" + hostname + '\'' + ", inetAddress=" + inetAddress + - ", nodeType=" + type + + ", ports=" + ports + '}'; } } @@ -197,12 +190,16 @@ public class Acl { return this; } - public Builder withTrustedNode(String hostname, String ipAddress, NodeType nodeType) { - return withTrustedNode(new Node(hostname, nodeType, ipAddress)); + public Builder withTrustedNode(String hostname, String ipAddress) { + return withTrustedNode(hostname, ipAddress, Set.of()); + } + + public Builder withTrustedNode(String hostname, String ipAddress, Set<Integer> ports) { + return withTrustedNode(new Node(hostname, ipAddress, ports)); } - public Builder withTrustedNode(String hostname, InetAddress inetAddress, NodeType nodeType) { - return withTrustedNode(new Node(hostname, nodeType, inetAddress)); + public Builder withTrustedNode(String hostname, InetAddress inetAddress, Set<Integer> ports) { + return withTrustedNode(new Node(hostname, inetAddress, ports)); } public Builder withTrustedPorts(Integer... ports) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 51111a66d10..34ff4feb548 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -98,7 +98,7 @@ public class RealNodeRepository implements NodeRepository { .collect(Collectors.groupingBy( GetAclResponse.Node::getTrustedBy, Collectors.mapping( - node -> new Acl.Node(node.hostname, NodeType.valueOf(node.nodeType), node.ipAddress), + node -> new Acl.Node(node.hostname, node.ipAddress, Set.copyOf(node.ports)), Collectors.toSet()))); // Group trusted networks by container hostname that trusts them diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java index 9afee6f7463..08d145b3ac8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java @@ -28,9 +28,9 @@ public class GetAclResponse { public GetAclResponse(@JsonProperty("trustedNodes") List<Node> trustedNodes, @JsonProperty("trustedNetworks") List<Network> trustedNetworks, @JsonProperty("trustedPorts") List<Port> trustedPorts) { - this.trustedNodes = trustedNodes == null ? List.of() : trustedNodes; - this.trustedNetworks = trustedNetworks == null ? List.of() : trustedNetworks; - this.trustedPorts = trustedPorts == null ? List.of() : trustedPorts; + this.trustedNodes = trustedNodes == null ? List.of() : List.copyOf(trustedNodes); + this.trustedNetworks = trustedNetworks == null ? List.of() : List.copyOf(trustedNetworks); + this.trustedPorts = trustedPorts == null ? List.of() : List.copyOf(trustedPorts); } @JsonIgnoreProperties(ignoreUnknown = true) @@ -45,16 +45,20 @@ public class GetAclResponse { @JsonProperty("ipAddress") public final String ipAddress; + @JsonProperty("ports") + public final List<Integer> ports; + @JsonProperty("trustedBy") public final String trustedBy; @JsonCreator public Node(@JsonProperty("hostname") String hostname, @JsonProperty("type") String nodeType, - @JsonProperty("ipAddress") String ipAddress, + @JsonProperty("ipAddress") String ipAddress, @JsonProperty("ports") List<Integer> ports, @JsonProperty("trustedBy") String trustedBy) { this.hostname = hostname; this.nodeType = nodeType; this.ipAddress = ipAddress; + this.ports = ports == null ? List.of() : List.copyOf(ports); this.trustedBy = trustedBy; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 435dc9cae85..cb2134b36af 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -55,8 +55,8 @@ public class AclMaintainer { if (context.isDisabled(NodeAgentTask.AclMaintainer)) return; // Apply acl to the filter table - editFlushOnError(context, IPVersion.IPv4, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv4, context.nodeType())); - editFlushOnError(context, IPVersion.IPv6, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv6, context.nodeType())); + editFlushOnError(context, IPVersion.IPv4, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv4)); + editFlushOnError(context, IPVersion.IPv6, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv6)); ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv4).ifPresent(addr -> applyRedirect(context, addr)); ipAddresses.getAddress(context.hostname().value(), IPVersion.IPv6).ifPresent(addr -> applyRedirect(context, addr)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java index 82dc388568b..462790b8d0f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.acl; -import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; @@ -23,8 +22,8 @@ class FilterTableLineEditor implements LineEditor { this.wantedRules = List.copyOf(wantedRules); } - static FilterTableLineEditor from(Acl acl, IPVersion ipVersion, NodeType nodeType) { - List<String> rules = acl.toRules(ipVersion, nodeType); + static FilterTableLineEditor from(Acl acl, IPVersion ipVersion) { + List<String> rules = acl.toRules(ipVersion); return new FilterTableLineEditor(rules); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java index c4bee8bb1dc..9fbe22482ea 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java @@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,17 +20,17 @@ public class AclTest { private static final Acl aclCommon = new Acl( Set.of(1234, 453), - testNodes("192.1.2.2", "fb00::1", "fe80::2", "fe80::3"), + testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2", "fe80::3"), Set.of()); private static final Acl aclWithoutPorts = new Acl( Set.of(), - testNodes("192.1.2.2", "fb00::1", "fe80::2"), + testNodes(Set.of(), "192.1.2.2", "fb00::1", "fe80::2"), Set.of()); @Test void no_trusted_ports() { - String listRulesIpv4 = String.join("\n", aclWithoutPorts.toRules(IPVersion.IPv4, NodeType.tenant)); + String listRulesIpv4 = String.join("\n", aclWithoutPorts.toRules(IPVersion.IPv4)); assertEquals( "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -44,7 +45,7 @@ public class AclTest { @Test void ipv4_rules() { - String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4, NodeType.tenant)); + String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4)); assertEquals( "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -52,7 +53,7 @@ public class AclTest { "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + "-A INPUT -i lo -j ACCEPT\n" + "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 1234,453 -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT\n" + "-A INPUT -s 192.1.2.2/32 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp-port-unreachable", listRulesIpv4); @@ -60,7 +61,7 @@ public class AclTest { @Test void ipv6_rules() { - String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6, NodeType.tenant)); + String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6)); assertEquals( "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -68,7 +69,7 @@ public class AclTest { "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + "-A INPUT -i lo -j ACCEPT\n" + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 1234,453 -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 453,1234 -j ACCEPT\n" + "-A INPUT -s fb00::1/128 -j ACCEPT\n" + "-A INPUT -s fe80::2/128 -j ACCEPT\n" + "-A INPUT -s fe80::3/128 -j ACCEPT\n" + @@ -79,17 +80,17 @@ public class AclTest { void ipv6_rules_stable_order() { Acl aclCommonDifferentOrder = new Acl( Set.of(453, 1234), - testNodes("fe80::2", "192.1.2.2", "fb00::1", "fe80::3"), + testNodes(Set.of(), "fe80::2", "192.1.2.2", "fb00::1", "fe80::3"), Set.of()); for (IPVersion ipVersion : IPVersion.values()) { - assertEquals(aclCommon.toRules(ipVersion, NodeType.tenant), aclCommonDifferentOrder.toRules(ipVersion, NodeType.tenant)); + assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion)); } } @Test void trusted_networks() { - Acl acl = new Acl(Set.of(4080), testNodes("127.0.0.1"), Set.of("10.0.0.0/24", "2001:db8::/32")); + Acl acl = new Acl(Set.of(4080), testNodes(Set.of(), "127.0.0.1"), Set.of("10.0.0.0/24", "2001:db8::/32")); assertEquals("-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -101,7 +102,7 @@ public class AclTest { "-A INPUT -s 127.0.0.1/32 -j ACCEPT\n" + "-A INPUT -s 10.0.0.0/24 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp-port-unreachable", - String.join("\n", acl.toRules(IPVersion.IPv4, NodeType.tenant))); + String.join("\n", acl.toRules(IPVersion.IPv4))); assertEquals("-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -112,12 +113,15 @@ public class AclTest { "-A INPUT -p tcp -m multiport --dports 4080 -j ACCEPT\n" + "-A INPUT -s 2001:db8::/32 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable", - String.join("\n", acl.toRules(IPVersion.IPv6, NodeType.tenant))); + String.join("\n", acl.toRules(IPVersion.IPv6))); } @Test void config_server_acl() { - Acl acl = new Acl(Set.of(22, 4443), testNodes(NodeType.config, "172.17.0.41", "172.17.0.42", "172.17.0.43"), Set.of()); + Set<Acl.Node> testNodes = Stream.concat(testNodes(NodeType.config, Set.of(), "172.17.0.41", "172.17.0.42", "172.17.0.43").stream(), + testNodes(NodeType.tenant, Set.of(19070), "172.17.0.81", "172.17.0.82", "172.17.0.83").stream()) + .collect(Collectors.toSet()); + Acl acl = new Acl(Set.of(22, 4443), testNodes, Set.of()); assertEquals(""" -P INPUT ACCEPT -P FORWARD ACCEPT @@ -126,17 +130,20 @@ public class AclTest { -A INPUT -i lo -j ACCEPT -A INPUT -p icmp -j ACCEPT -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 172.17.0.41/32 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 172.17.0.42/32 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 172.17.0.43/32 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -p tcp -m multiport --dports 2181,2182,2183 -j REJECT --reject-with icmp-port-unreachable -A INPUT -s 172.17.0.41/32 -j ACCEPT -A INPUT -s 172.17.0.42/32 -j ACCEPT -A INPUT -s 172.17.0.43/32 -j ACCEPT + -A INPUT -s 172.17.0.81/32 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 172.17.0.82/32 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 172.17.0.83/32 -p tcp -m multiport --dports 19070 -j ACCEPT -A INPUT -j REJECT --reject-with icmp-port-unreachable""", - String.join("\n", acl.toRules(IPVersion.IPv4, NodeType.config))); + String.join("\n", acl.toRules(IPVersion.IPv4))); + + Set<Acl.Node> testNodes2 = Stream.concat(testNodes(NodeType.config, Set.of(), "2001:db8::41", "2001:db8::42", "2001:db8::43").stream(), + testNodes(NodeType.tenant, Set.of(19070), "2001:db8::81", "2001:db8::82", "2001:db8::83").stream()) + .collect(Collectors.toSet()); + Acl acl2 = new Acl(Set.of(22, 4443), testNodes2, Set.of()); - Acl acl2 = new Acl(Set.of(22, 4443), testNodes(NodeType.config, "2001:db8::41", "2001:db8::42", "2001:db8::43"), Set.of()); assertEquals(""" -P INPUT ACCEPT -P FORWARD ACCEPT @@ -145,24 +152,23 @@ public class AclTest { -A INPUT -i lo -j ACCEPT -A INPUT -p ipv6-icmp -j ACCEPT -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 2001:db8::41/128 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 2001:db8::42/128 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 2001:db8::43/128 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -p tcp -m multiport --dports 2181,2182,2183 -j REJECT --reject-with icmp6-port-unreachable -A INPUT -s 2001:db8::41/128 -j ACCEPT -A INPUT -s 2001:db8::42/128 -j ACCEPT -A INPUT -s 2001:db8::43/128 -j ACCEPT + -A INPUT -s 2001:db8::81/128 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 2001:db8::82/128 -p tcp -m multiport --dports 19070 -j ACCEPT + -A INPUT -s 2001:db8::83/128 -p tcp -m multiport --dports 19070 -j ACCEPT -A INPUT -j REJECT --reject-with icmp6-port-unreachable""", - String.join("\n", acl2.toRules(IPVersion.IPv6, NodeType.config))); + String.join("\n", acl2.toRules(IPVersion.IPv6))); } - private static Set<Acl.Node> testNodes(String... address) { - return testNodes(NodeType.tenant, address); + private static Set<Acl.Node> testNodes(Set<Integer> ports, String... address) { + return testNodes(NodeType.tenant, ports, address); } - private static Set<Acl.Node> testNodes(NodeType nodeType, String... address) { + private static Set<Acl.Node> testNodes(NodeType nodeType, Set<Integer> ports, String... address) { return Arrays.stream(address) - .map(addr -> new Acl.Node("hostname", nodeType, addr)) + .map(addr -> new Acl.Node("hostname", addr, ports)) .collect(Collectors.toUnmodifiableSet()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index 9d5683c9856..827c6ebb6ec 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -48,10 +48,10 @@ public class AclMaintainerTest { @Test void configures_full_container_acl_from_empty() { Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) - .withTrustedNode("hostname1", "3001::abcd", NodeType.tenant) - .withTrustedNode("hostname2", "3001::1234", NodeType.tenant) - .withTrustedNode("hostname1", "192.168.0.5", NodeType.tenant) - .withTrustedNode("hostname4", "172.16.5.234", NodeType.tenant).build(); + .withTrustedNode("hostname1", "3001::abcd") + .withTrustedNode("hostname2", "3001::1234") + .withTrustedNode("hostname1", "192.168.0.5") + .withTrustedNode("hostname4", "172.16.5.234").build(); NodeAgentContext context = contextGenerator.apply(acl); ipAddresses.addAddress(context.hostname().value(), "2001::1"); @@ -163,7 +163,7 @@ public class AclMaintainerTest { @Test void only_configure_iptables_for_ipversion_that_differs() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd", NodeType.tenant).build(); + Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); NodeAgentContext context = contextGenerator.apply(acl); ipAddresses.addAddress(context.hostname().value(), "2001::1"); @@ -209,7 +209,7 @@ public class AclMaintainerTest { @Test void rollback_is_attempted_when_applying_acl_fail() { - Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd", NodeType.tenant).build(); + Acl acl = new Acl.Builder().withTrustedPorts(22, 4443).withTrustedNode("hostname1", "3001::abcd").build(); NodeAgentContext context = contextGenerator.apply(acl); ipAddresses.addAddress(context.hostname().value(), "2001::1"); @@ -248,12 +248,12 @@ public class AclMaintainerTest { @Test public void config_server_acl() { Acl acl = new Acl.Builder().withTrustedPorts(22, 4443) - .withTrustedNode("cfg1", "2001:db8::1", NodeType.config) - .withTrustedNode("cfg2", "2001:db8::2", NodeType.config) - .withTrustedNode("cfg3", "2001:db8::3", NodeType.config) - .withTrustedNode("cfg1", "172.17.0.41", NodeType.config) - .withTrustedNode("cfg2", "172.17.0.42", NodeType.config) - .withTrustedNode("cfg3", "172.17.0.43", NodeType.config) + .withTrustedNode("cfg1", "2001:db8::1") + .withTrustedNode("cfg2", "2001:db8::2") + .withTrustedNode("cfg3", "2001:db8::3") + .withTrustedNode("cfg1", "172.17.0.41") + .withTrustedNode("cfg2", "172.17.0.42") + .withTrustedNode("cfg3", "172.17.0.43") .build(); NodeAgentContext context = NodeAgentContextImpl.builder("cfg3.example.com") .fileSystem(fileSystem) @@ -287,10 +287,6 @@ public class AclMaintainerTest { -A INPUT -i lo -j ACCEPT -A INPUT -p icmp -j ACCEPT -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 172.17.0.41/32 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 172.17.0.42/32 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 172.17.0.43/32 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -p tcp -m multiport --dports 2181,2182,2183 -j REJECT --reject-with icmp-port-unreachable -A INPUT -s 172.17.0.41/32 -j ACCEPT -A INPUT -s 172.17.0.42/32 -j ACCEPT -A INPUT -s 172.17.0.43/32 -j ACCEPT @@ -307,10 +303,6 @@ public class AclMaintainerTest { -A INPUT -i lo -j ACCEPT -A INPUT -p ipv6-icmp -j ACCEPT -A INPUT -p tcp -m multiport --dports 22,4443 -j ACCEPT - -A INPUT -s 2001:db8::1/128 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 2001:db8::2/128 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -s 2001:db8::3/128 -p tcp -m multiport --dports 2181,2182,2183 -j ACCEPT - -A INPUT -p tcp -m multiport --dports 2181,2182,2183 -j REJECT --reject-with icmp6-port-unreachable -A INPUT -s 2001:db8::1/128 -j ACCEPT -A INPUT -s 2001:db8::2/128 -j ACCEPT -A INPUT -s 2001:db8::3/128 -j ACCEPT diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java index 39d1a46f198..9263a1a8dd1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java @@ -19,7 +19,7 @@ public class FilterTableLineEditorTest { @Test void filter_set_wanted_rules() { - Acl acl = new Acl.Builder().withTrustedPorts(22).withTrustedNode("hostname", "3001::1", NodeType.tenant).build(); + Acl acl = new Acl.Builder().withTrustedPorts(22).withTrustedNode("hostname", "3001::1").build(); assertFilterTableLineEditorResult( acl, IPVersion.IPv6, @@ -60,7 +60,7 @@ public class FilterTableLineEditorTest { private static void assertFilterTableLineEditorResult( Acl acl, IPVersion ipVersion, String currentFilterTable, String expectedRestoreFileContent) { - FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion, NodeType.tenant); + FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion); Editor editor = new Editor( "nat-table", () -> List.of(currentFilterTable.split("\n")), @@ -72,16 +72,17 @@ public class FilterTableLineEditorTest { private static void assertFilterTableDiff(List<Integer> currentIpSuffix, List<Integer> wantedIpSuffix, String diff) { Acl.Builder currentAcl = new Acl.Builder(); NodeType nodeType = NodeType.tenant; - currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i, nodeType)); + currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i)); List<String> currentTable = new ArrayList<>(); Acl.Builder wantedAcl = new Acl.Builder(); - wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i, nodeType)); + wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i)); - new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6, nodeType)) + new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6)) .edit(log -> {}); - new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6, nodeType)) + new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6)) .edit(log -> assertEquals(diff, log)); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index c2d4506a28c..14fa2e1c8ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -40,7 +40,7 @@ public class History { .stream() .sorted(Comparator.comparing(Event::at)) .skip(Math.max(log.size() - maxLogSize, 0)) - .collect(Collectors.toUnmodifiableList()); + .toList(); this.maxLogSize = maxLogSize; } @@ -53,7 +53,7 @@ public class History { /** Returns the age of this node as best as we can determine: The time since the first event registered for it */ public Duration age(Instant now) { - Instant oldestEventTime = events.values().stream().map(event -> event.at()).sorted().findFirst().orElse(now); + Instant oldestEventTime = events.values().stream().map(Event::at).sorted().findFirst().orElse(now); return Duration.between(oldestEventTime, now); } @@ -117,19 +117,18 @@ public class History { public History recordStateTransition(Node.State from, Node.State to, Agent agent, Instant at) { // If the event is a re-reservation, allow the new one to override the older one. if (from == to && from != Node.State.reserved) return this; - switch (to) { - case provisioned: return this.with(new Event(Event.Type.provisioned, agent, at)); - case deprovisioned: return this.with(new Event(Event.Type.deprovisioned, agent, at)); - case ready: return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at)); - case active: return this.with(new Event(Event.Type.activated, agent, at)); - case inactive: return this.with(new Event(Event.Type.deactivated, agent, at)); - case reserved: return this.with(new Event(Event.Type.reserved, agent, at)); - case failed: return this.with(new Event(Event.Type.failed, agent, at)); - case dirty: return this.with(new Event(Event.Type.deallocated, agent, at)); - case parked: return this.with(new Event(Event.Type.parked, agent, at)); - case breakfixed: return this.with(new Event(Event.Type.breakfixed, agent, at)); - default: return this; - } + return switch (to) { + case provisioned -> this.with(new Event(Event.Type.provisioned, agent, at)); + case deprovisioned -> this.with(new Event(Event.Type.deprovisioned, agent, at)); + case ready -> this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at)); + case active -> this.with(new Event(Event.Type.activated, agent, at)); + case inactive -> this.with(new Event(Event.Type.deactivated, agent, at)); + case reserved -> this.with(new Event(Event.Type.reserved, agent, at)); + case failed -> this.with(new Event(Event.Type.failed, agent, at)); + case dirty -> this.with(new Event(Event.Type.deallocated, agent, at)); + case parked -> this.with(new Event(Event.Type.parked, agent, at)); + case breakfixed -> this.with(new Event(Event.Type.breakfixed, agent, at)); + }; } /** @@ -154,17 +153,7 @@ public class History { } /** An event which may happen to a node */ - public static class Event { - - private final Instant at; - private final Agent agent; - private final Type type; - - public Event(Event.Type type, Agent agent, Instant at) { - this.type = type; - this.agent = agent; - this.at = at; - } + public record Event(Type type, Agent agent, Instant at) { public enum Type { // State changes @@ -213,7 +202,7 @@ public class History { this.applicationLevel = applicationLevel; } - /** Returns true if this is an application level event and false it it is a node level event */ + /** Returns true if this is an application-level event and false if it's a node-level event */ public boolean isApplicationLevel() { return applicationLevel; } } @@ -229,19 +218,6 @@ public class History { @Override public String toString() { return "'" + type + "' event at " + at + " by " + agent; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Event event = (Event) o; - return at.equals(event.at) && agent == event.agent && type == event.type; - } - - @Override - public int hashCode() { - return Objects.hash(at, agent, type); - } - } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index afee9a55870..43843f6fe5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -78,7 +78,9 @@ public class RetiringOsUpgrader implements OsUpgrader { /** The duration this leaves new nodes alone before scheduling any upgrade */ private Duration gracePeriod() { - return nodeRepository.zone().system().isCd() ? Duration.ofDays(1) : Duration.ofDays(30); + if (nodeRepository.zone().system().isCd()) return Duration.ofDays(1); + if (!nodeRepository.zone().environment().isProduction()) return Duration.ofDays(7); + return Duration.ofDays(30); } } diff --git a/persistence/src/Doxyfile b/persistence/src/Doxyfile deleted file mode 100644 index 0ae14d0acc1..00000000000 --- a/persistence/src/Doxyfile +++ /dev/null @@ -1,994 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.2.18 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# General configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = Storage - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../doc - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, -# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en -# (Japanese with english messages), Korean, Norwegian, Polish, Portuguese, -# Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish and Ukrainian. - -OUTPUT_LANGUAGE = English - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these class will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower case letters. If set to YES upper case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are adviced to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explict @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# reimplements. - -INHERIT_DOCS = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consist of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = YES - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = storage - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl - -FILE_PATTERNS = *.h *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet - -HTML_STYLESHEET = ../cpp/vespa_link.css - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output dir. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non empty doxygen will try to run -# the html help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the Html help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript and frames is required (for instance Mozilla, Netscape 4.0+, -# or Internet explorer 4.0+). Note that for large projects the tree generation -# can take a very long time. In such cases it is better to disable this feature. -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimised for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assigments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_XML = NO - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_PREDEF_ONLY tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES tag can be used to specify one or more tagfiles. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in Html, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superceded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yield more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermedate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO - -# The CGI_NAME tag should be the name of the CGI script that -# starts the search engine (doxysearch) with the correct parameters. -# A script with this name will be generated by doxygen. - -CGI_NAME = search.cgi - -# The CGI_URL tag should be the absolute URL to the directory where the -# cgi binaries are located. See the documentation of your http daemon for -# details. - -CGI_URL = - -# The DOC_URL tag should be the absolute URL to the directory where the -# documentation is located. If left blank the absolute path to the -# documentation, with file:// prepended to it, will be used. - -DOC_URL = - -# The DOC_ABSPATH tag should be the absolute path to the directory where the -# documentation is located. If left blank the directory on the local machine -# will be used. - -DOC_ABSPATH = - -# The BIN_ABSPATH tag must point to the directory where the doxysearch binary -# is installed. - -BIN_ABSPATH = /usr/local/bin/ - -# The EXT_DOC_PATHS tag can be used to specify one or more paths to -# documentation generated for other projects. This allows doxysearch to search -# the documentation for these projects as well. - -EXT_DOC_PATHS = diff --git a/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp b/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp index 78486f54704..bd26323deb4 100644 --- a/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp +++ b/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp @@ -269,100 +269,60 @@ std::string delayed_split_query_tree_dump = //----------------------------------------------------------------------------- -Node::UP optimize(Node::UP root, bool white_list, bool split, bool delay) { - return UnpackingIteratorsOptimizer::optimize(std::move(root), white_list, split, delay); +Node::UP optimize(Node::UP root, bool white_list, bool split) { + return UnpackingIteratorsOptimizer::optimize(std::move(root), white_list, split); } TEST(UnpackingIteratorsOptimizerTest, require_that_root_phrase_node_can_be_left_alone) { - std::string actual1 = dump_query(*optimize(make_phrase(), false, false, false)); - std::string actual2 = dump_query(*optimize(make_phrase(), false, true, false)); - std::string actual3 = dump_query(*optimize(make_phrase(), true, false, false)); + std::string actual1 = dump_query(*optimize(make_phrase(), false, false)); + std::string actual2 = dump_query(*optimize(make_phrase(), false, true)); + std::string actual3 = dump_query(*optimize(make_phrase(), true, false)); std::string expect = plain_phrase_dump; EXPECT_EQ(actual1, expect); EXPECT_EQ(actual2, expect); EXPECT_EQ(actual3, expect); } -TEST(UnpackingIteratorsOptimizerTest, require_that_root_phrase_node_can_be_delayed) { - std::string actual1 = dump_query(*optimize(make_phrase(), false, false, true)); - std::string actual2 = dump_query(*optimize(make_phrase(), false, true, true)); - std::string actual3 = dump_query(*optimize(make_phrase(), true, false, true)); - std::string expect = delayed_phrase_dump; - EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); - EXPECT_EQ(actual3, expect); -} - TEST(UnpackingIteratorsOptimizerTest, require_that_root_phrase_node_can_be_split) { - std::string actual1 = dump_query(*optimize(make_phrase(), true, true, true)); - std::string actual2 = dump_query(*optimize(make_phrase(), true, true, false)); + std::string actual1 = dump_query(*optimize(make_phrase(), true, true)); std::string expect = split_phrase_dump; EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); } //----------------------------------------------------------------------------- TEST(UnpackingIteratorsOptimizerTest, require_that_root_same_element_node_can_be_left_alone) { - std::string actual1 = dump_query(*optimize(make_same_element(), false, false, false)); - std::string actual2 = dump_query(*optimize(make_same_element(), false, true, false)); - std::string actual3 = dump_query(*optimize(make_same_element(), true, false, false)); + std::string actual1 = dump_query(*optimize(make_same_element(), false, false)); + std::string actual2 = dump_query(*optimize(make_same_element(), false, true)); + std::string actual3 = dump_query(*optimize(make_same_element(), true, false)); std::string expect = plain_same_element_dump; EXPECT_EQ(actual1, expect); EXPECT_EQ(actual2, expect); EXPECT_EQ(actual3, expect); } -TEST(UnpackingIteratorsOptimizerTest, require_that_root_same_element_node_can_be_delayed) { - std::string actual1 = dump_query(*optimize(make_same_element(), false, false, true)); - std::string actual2 = dump_query(*optimize(make_same_element(), false, true, true)); - std::string actual3 = dump_query(*optimize(make_same_element(), true, false, true)); - std::string expect = delayed_same_element_dump; - EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); - EXPECT_EQ(actual3, expect); -} - TEST(UnpackingIteratorsOptimizerTest, require_that_root_same_element_node_can_be_split) { - std::string actual1 = dump_query(*optimize(make_same_element(), true, true, true)); - std::string actual2 = dump_query(*optimize(make_same_element(), true, true, false)); + std::string actual1 = dump_query(*optimize(make_same_element(), true, true)); std::string expect = split_same_element_dump; EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); } //----------------------------------------------------------------------------- TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_left_alone) { - std::string actual1 = dump_query(*optimize(make_query_tree(), false, false, false)); - std::string actual2 = dump_query(*optimize(make_query_tree(), true, false, false)); + std::string actual1 = dump_query(*optimize(make_query_tree(), false, false)); + std::string actual2 = dump_query(*optimize(make_query_tree(), true, false)); std::string expect = plain_query_tree_dump; EXPECT_EQ(actual1, expect); EXPECT_EQ(actual2, expect); } -TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_delayed) { - std::string actual1 = dump_query(*optimize(make_query_tree(), false, false, true)); - std::string actual2 = dump_query(*optimize(make_query_tree(), true, false, true)); - std::string expect = delayed_query_tree_dump; - EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); -} - TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_split) { - std::string actual1 = dump_query(*optimize(make_query_tree(), false, true, false)); - std::string actual2 = dump_query(*optimize(make_query_tree(), true, true, false)); + std::string actual1 = dump_query(*optimize(make_query_tree(), false, true)); + std::string actual2 = dump_query(*optimize(make_query_tree(), true, true)); std::string expect = split_query_tree_dump; EXPECT_EQ(actual1, expect); EXPECT_EQ(actual2, expect); } -TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_delayed_and_split) { - std::string actual1 = dump_query(*optimize(make_query_tree(), false, true, true)); - std::string actual2 = dump_query(*optimize(make_query_tree(), true, true, true)); - std::string expect = delayed_split_query_tree_dump; - EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); -} - GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index f82b4c9243f..f1183c2556a 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -3,13 +3,9 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchlib/engine/searchreply.h> -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/util/compressor.h> -#include <vespa/searchsummary/docsummary/docsumwriter.h> -#include <vespa/metrics/metricset.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 7b4db47f585..942dac63955 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -84,7 +84,7 @@ DocsumContext::createSlimeReply() if (_request.expired() ) { break; } Cursor &docSumC = array.addObject(); ObjectSymbolInserter inserter(docSumC, docsumSym); - if ((docId != search::endDocId) && !rci.mustSkip) { + if ((docId != search::endDocId) && rci.outputClass != nullptr) { _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, inserter); } num_ok++; @@ -103,7 +103,7 @@ DocsumContext::createSlimeReply() DocsumContext::DocsumContext(const DocsumRequest & request, IDocsumWriter & docsumWriter, IDocsumStore & docsumStore, std::shared_ptr<Matcher> matcher, ISearchContext & searchCtx, IAttributeContext & attrCtx, - IAttributeManager & attrMgr, SessionManager & sessionMgr) : + const IAttributeManager & attrMgr, SessionManager & sessionMgr) : _request(request), _docsumWriter(docsumWriter), _docsumStore(docsumStore), @@ -124,24 +124,24 @@ DocsumContext::getDocsums() } void -DocsumContext::FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment *) +DocsumContext::FillSummaryFeatures(search::docsummary::GetDocsumsState& state) { - assert(&_docsumState == state); + assert(&_docsumState == &state); if (_matcher->canProduceSummaryFeatures()) { - state->_summaryFeatures = _matcher->getSummaryFeatures(_request, _searchCtx, _attrCtx, _sessionMgr); + state._summaryFeatures = _matcher->getSummaryFeatures(_request, _searchCtx, _attrCtx, _sessionMgr); } - state->_summaryFeaturesCached = false; + state._summaryFeaturesCached = false; } void -DocsumContext::FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment *) +DocsumContext::FillRankFeatures(search::docsummary::GetDocsumsState& state) { - assert(&_docsumState == state); + assert(&_docsumState == &state); // check if we are allowed to run - if ( ! state->_args.dumpFeatures()) { + if ( ! state._args.dumpFeatures()) { return; } - state->_rankFeatures = _matcher->getRankFeatures(_request, _searchCtx, _attrCtx, _sessionMgr); + state._rankFeatures = _matcher->getRankFeatures(_request, _searchCtx, _attrCtx, _sessionMgr); } std::unique_ptr<MatchingElements> diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h index 958e19f9bed..5c1db91f05d 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h @@ -27,7 +27,7 @@ private: std::shared_ptr<matching::Matcher> _matcher; matching::ISearchContext & _searchCtx; search::attribute::IAttributeContext & _attrCtx; - search::IAttributeManager & _attrMgr; + const search::IAttributeManager & _attrMgr; search::docsummary::GetDocsumsState _docsumState; matching::SessionManager & _sessionMgr; @@ -43,14 +43,14 @@ public: std::shared_ptr<matching::Matcher> matcher, matching::ISearchContext & searchCtx, search::attribute::IAttributeContext & attrCtx, - search::IAttributeManager & attrMgr, + const search::IAttributeManager & attrMgr, matching::SessionManager & sessionMgr); search::engine::DocsumReply::UP getDocsums(); // Implements GetDocsumsStateCallback - void FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; - void FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; + void FillSummaryFeatures(search::docsummary::GetDocsumsState& state) override; + void FillRankFeatures(search::docsummary::GetDocsumsState& state) override; std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields &fields) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h index e3c4705104f..946c45feb4b 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/isummarymanager.h @@ -31,16 +31,11 @@ public: typedef std::unique_ptr<ISummarySetup> UP; typedef std::shared_ptr<ISummarySetup> SP; - virtual ~ISummarySetup() {} + ~ISummarySetup() override = default; virtual search::docsummary::IDocsumWriter &getDocsumWriter() const = 0; virtual const search::docsummary::ResultConfig &getResultConfig() = 0; virtual search::docsummary::IDocsumStore::UP createDocsumStore() = 0; - - // Inherit doc from IDocsumEnvironment - virtual search::IAttributeManager *getAttributeManager() override = 0; - virtual vespalib::string lookupIndex(const vespalib::string & s) const override = 0; - virtual juniper::Juniper *getJuniper() override = 0; }; typedef std::unique_ptr<ISummaryManager> UP; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 18c84038f9b..3e3a3529e46 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -10,6 +10,7 @@ #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchsummary/docsummary/docsumconfig.h> +#include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/fastlib/text/normwordfolder.h> @@ -89,7 +90,7 @@ SummarySetup(const vespalib::string & baseDir, const SummaryConfig & summaryCfg, _juniperConfig(), _attributeMgr(std::move(attributeMgr)), _docStore(std::move(docStore)), - _repo(repo) + _repo(std::move(repo)) { auto resultConfig = std::make_unique<ResultConfig>(); if (!resultConfig->ReadConfig(summaryCfg, make_string("SummaryManager(%s)", baseDir.c_str()).c_str())) { @@ -103,7 +104,7 @@ SummarySetup(const vespalib::string & baseDir, const SummaryConfig & summaryCfg, _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, _wordFolder.get()); _docsumWriter = std::make_unique<DynamicDocsumWriter>(std::move(resultConfig), std::unique_ptr<KeywordExtractor>()); - DynamicDocsumConfig dynCfg(this, _docsumWriter.get()); + DynamicDocsumConfig dynCfg(*this, _docsumWriter.get()); dynCfg.configure(summarymapCfg); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index 0fd45bb28fb..8be949a7351 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -43,9 +43,9 @@ public: search::docsummary::IDocsumStore::UP createDocsumStore() override; - search::IAttributeManager * getAttributeManager() override { return _attributeMgr.get(); } + const search::IAttributeManager * getAttributeManager() const override { return _attributeMgr.get(); } vespalib::string lookupIndex(const vespalib::string & s) const override { (void) s; return ""; } - juniper::Juniper * getJuniper() override { return _juniperConfig.get(); } + const juniper::Juniper * getJuniper() const override { return _juniperConfig.get(); } }; private: diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 427d4507c43..eefb8411df2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -24,6 +24,21 @@ using vespalib::Issue; namespace { +struct LazyThreadTraceInserter { + search::engine::Trace &root_trace; + std::unique_ptr<vespalib::slime::Inserter> inserter; + LazyThreadTraceInserter(search::engine::Trace &root_trace_in) + : root_trace(root_trace_in), inserter() {} + void handle(const search::engine::Trace &thread_trace) { + if (thread_trace.hasTrace()) { + if (!inserter) { + inserter = std::make_unique<vespalib::slime::ArrayInserter>(root_trace.createCursor("query_execution").setArray("threads")); + } + vespalib::slime::inject(thread_trace.getRoot(), *inserter); + } + } +}; + struct TimedMatchLoopCommunicator final : IMatchLoopCommunicator { IMatchLoopCommunicator &communicator; vespalib::Timer timer; @@ -104,17 +119,12 @@ MatchMaster::match(search::engine::Trace & trace, double query_time_s = vespalib::to_s(query_latency_time.elapsed()); double rerank_time_s = vespalib::to_s(timedCommunicator.elapsed); double match_time_s = 0.0; - std::unique_ptr<vespalib::slime::Inserter> inserter; - if (trace.shouldTrace(4)) { - inserter = std::make_unique<vespalib::slime::ArrayInserter>(trace.createCursor("query_execution").setArray("threads")); - } + LazyThreadTraceInserter inserter(trace); for (size_t i = 0; i < threadState.size(); ++i) { const MatchThread & matchThread = *threadState[i]; match_time_s = std::max(match_time_s, matchThread.get_match_time()); _stats.merge_partition(matchThread.get_thread_stats(), i); - if (inserter && matchThread.getTrace().hasTrace()) { - vespalib::slime::inject(matchThread.getTrace().getRoot(), *inserter); - } + inserter.handle(matchThread.getTrace()); matchThread.get_issues().for_each_message([](const auto &msg){ Issue::report(Issue(msg)); }); } _stats.queryLatency(query_time_s); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index a2af40aae96..6fecdbf611c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -187,8 +187,7 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _query.setWhiteListBlueprint(metaStore.createWhiteListBlueprint()); trace.addEvent(5, "Deserialize and build query tree"); _valid = _query.buildTree(queryStack, location, viewResolver, indexEnv, - rankSetup.split_unpacking_iterators(), - rankSetup.delay_unpacking_iterators()); + SplitUnpackingIterators::check(_queryEnv.getProperties(), rankSetup.split_unpacking_iterators())); if (_valid) { _query.extractTerms(_queryEnv.terms()); _query.extractLocations(_queryEnv.locations()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 5802433ae02..aaacb971ee0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -164,7 +164,7 @@ Query::~Query() = default; bool Query::buildTree(vespalib::stringref stack, const string &location, const ViewResolver &resolver, const IIndexEnvironment &indexEnv, - bool split_unpacking_iterators, bool delay_unpacking_iterators) + bool split_unpacking_iterators) { SimpleQueryStackDumpIterator stack_dump_iterator(stack); _query_tree = QueryTreeCreator<ProtonNodeTypes>::create(stack_dump_iterator); @@ -173,7 +173,7 @@ Query::buildTree(vespalib::stringref stack, const string &location, _query_tree->accept(prefixSameElementSubIndexes); exchange_location_nodes(location, _query_tree, _locations); _query_tree = UnpackingIteratorsOptimizer::optimize(std::move(_query_tree), - bool(_whiteListBlueprint), split_unpacking_iterators, delay_unpacking_iterators); + bool(_whiteListBlueprint), split_unpacking_iterators); ResolveViewVisitor resolve_visitor(resolver, indexEnv); _query_tree->accept(resolve_visitor); return true; diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index 09eaed5c4a9..a666e382485 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -53,8 +53,7 @@ public: const vespalib::string &location, const ViewResolver &resolver, const search::fef::IIndexEnvironment &idxEnv, - bool split_unpacking_iterators = false, - bool delay_unpacking_iterators = false); + bool split_unpacking_iterators = false); /** * Extract query terms from the query tree; to be used to build diff --git a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp index 72e0153b5c2..e3ab2c9faa8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp @@ -72,13 +72,9 @@ struct TermExpander : QueryVisitor { struct NodeTraverser : TemplateTermVisitor<NodeTraverser, ProtonNodeTypes> { bool split_unpacking_iterators; - bool delay_unpacking_iterators; - - NodeTraverser(bool split_unpacking_iterators_in, - bool delay_unpacking_iterators_in) - : split_unpacking_iterators(split_unpacking_iterators_in), - delay_unpacking_iterators(delay_unpacking_iterators_in) {} + NodeTraverser(bool split_unpacking_iterators_in) + : split_unpacking_iterators(split_unpacking_iterators_in) {} template <class TermNode> void visitTerm(TermNode &) {} void visit(ProtonNodeTypes::And &n) override { for (Node *child: n.getChildren()) { @@ -92,16 +88,6 @@ struct NodeTraverser : TemplateTermVisitor<NodeTraverser, ProtonNodeTypes> expander.flush(n); } } - void visit(ProtonNodeTypes::Phrase &n) override { - if (delay_unpacking_iterators) { - n.set_expensive(true); - } - } - void visit(ProtonNodeTypes::SameElement &n) override { - if (delay_unpacking_iterators) { - n.set_expensive(true); - } - } }; } // namespace proton::matching::<unnamed> @@ -109,12 +95,10 @@ struct NodeTraverser : TemplateTermVisitor<NodeTraverser, ProtonNodeTypes> search::query::Node::UP UnpackingIteratorsOptimizer::optimize(search::query::Node::UP root, bool has_white_list, - bool split_unpacking_iterators, - bool delay_unpacking_iterators) + bool split_unpacking_iterators) { - if (split_unpacking_iterators || delay_unpacking_iterators) { - NodeTraverser traverser(split_unpacking_iterators, - delay_unpacking_iterators); + if (split_unpacking_iterators) { + NodeTraverser traverser(split_unpacking_iterators); root->accept(traverser); } if (has_white_list && split_unpacking_iterators) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h index 3955c3cbc64..d4bfadf6c31 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h +++ b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h @@ -14,8 +14,7 @@ namespace proton::matching { struct UnpackingIteratorsOptimizer { static search::query::Node::UP optimize(search::query::Node::UP root, bool has_white_list, - bool split_unpacking_iterators, - bool delay_unpacking_iterators); + bool split_unpacking_iterators); }; } diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index f9b6271b47b..f39018846a0 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -186,7 +186,6 @@ vespa_define_module( src/tests/nearsearch src/tests/postinglistbm src/tests/predicate - src/tests/prettyfloat src/tests/query src/tests/queryeval src/tests/queryeval/blueprint diff --git a/searchlib/src/Doxyfile b/searchlib/src/Doxyfile deleted file mode 100644 index 96544cd14d5..00000000000 --- a/searchlib/src/Doxyfile +++ /dev/null @@ -1,1162 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.3.9.1 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = SearchLib - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of source -# files, where putting all generated files in the same directory would otherwise -# cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, -# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, -# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, -# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, -# Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is used -# as the annotated text. Otherwise, the brief description is used as-is. If left -# blank, the following values are used ("$name" is automatically replaced with the -# name of the entity): "The $name class" "The $name widget" "The $name file" -# "is" "provides" "specifies" "contains" "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. - -SHOW_DIRECTORIES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = searchlib - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm - -FILE_PATTERNS = - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = IAM_DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superseded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes that -# lay further from the root node will be omitted. Note that setting this option to -# 1 or 2 may greatly reduce the computation time needed for large code bases. Also -# note that a graph may be further truncated if the graph's image dimensions are -# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). -# If 0 is used for the depth value (the default), the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp b/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp index 06ca299b9ba..3029f2c01a5 100644 --- a/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp +++ b/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp @@ -120,7 +120,7 @@ TEST(ElementIteratorTest, require_that_non_searchcontext) fef::TermFieldMatchDataArray tfmda; tfmda.add(&tfmd); queryeval::FakeResult result = createResult(); - auto search = std::make_unique<queryeval::FakeSearch>("","","", result, tfmda); + auto search = std::make_unique<queryeval::FakeSearch>("","","", result, std::move(tfmda)); queryeval::ElementIteratorWrapper wrapper(std::move(search), tfmd); verifyElementIterator(wrapper); } diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index 8400eb2b27a..e61c372d415 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -280,19 +280,11 @@ TEST("test stuff") { } { // vespa.matching.split_unpacking_iterators EXPECT_EQUAL(matching::SplitUnpackingIterators::NAME, vespalib::string("vespa.matching.split_unpacking_iterators")); - EXPECT_EQUAL(matching::SplitUnpackingIterators::DEFAULT_VALUE, false); + EXPECT_TRUE(matching::SplitUnpackingIterators::DEFAULT_VALUE); Properties p; - EXPECT_EQUAL(matching::SplitUnpackingIterators::check(p), false); - p.add("vespa.matching.split_unpacking_iterators", "true"); - EXPECT_EQUAL(matching::SplitUnpackingIterators::check(p), true); - } - { // vespa.matching.delay_unpacking_iterators - EXPECT_EQUAL(matching::DelayUnpackingIterators::NAME, vespalib::string("vespa.matching.delay_unpacking_iterators")); - EXPECT_EQUAL(matching::DelayUnpackingIterators::DEFAULT_VALUE, false); - Properties p; - EXPECT_EQUAL(matching::DelayUnpackingIterators::check(p), false); - p.add("vespa.matching.delay_unpacking_iterators", "true"); - EXPECT_EQUAL(matching::DelayUnpackingIterators::check(p), true); + EXPECT_TRUE(matching::SplitUnpackingIterators::check(p)); + p.add("vespa.matching.split_unpacking_iterators", "false"); + EXPECT_FALSE(matching::SplitUnpackingIterators::check(p)); } { // vespa.matching.termwise_limit EXPECT_EQUAL(matching::TermwiseLimit::NAME, vespalib::string("vespa.matching.termwise_limit")); diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index ca3596e7a97..f42ff600f78 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -20,12 +20,12 @@ struct State { ~State(); void setArray(TermFieldMatchDataArray value) { - array = value; + array = std::move(value); } }; -State::State() : term(), md(), f3(0), f5(0), f7(0), array() {} -State::~State() {} +State::State() : term(), md(), f3(nullptr), f5(nullptr), f7(nullptr), array() {} +State::~State() = default; void testInvalidId() { const TermFieldMatchData empty; diff --git a/searchlib/src/tests/prettyfloat/.gitignore b/searchlib/src/tests/prettyfloat/.gitignore deleted file mode 100644 index bf0327f3372..00000000000 --- a/searchlib/src/tests/prettyfloat/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -prettyfloat_test -searchlib_prettyfloat_test_app diff --git a/searchlib/src/tests/prettyfloat/CMakeLists.txt b/searchlib/src/tests/prettyfloat/CMakeLists.txt deleted file mode 100644 index 907b7661800..00000000000 --- a/searchlib/src/tests/prettyfloat/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_prettyfloat_test_app TEST - SOURCES - prettyfloat.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_prettyfloat_test_app COMMAND searchlib_prettyfloat_test_app) diff --git a/searchlib/src/tests/prettyfloat/prettyfloat.cpp b/searchlib/src/tests/prettyfloat/prettyfloat.cpp deleted file mode 100644 index a84ac59c964..00000000000 --- a/searchlib/src/tests/prettyfloat/prettyfloat.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("prettyfloat_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/searchlib/common/hitrank.h> - -using namespace search; - -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("prettyfloat_test"); - { - RawBuf buf(5000); - SignedHitRank rank = 10; - buf.addSignedHitRank(rank); - *buf.GetWritableFillPos() = '\0'; - EXPECT_EQUAL(std::string("10"), buf.GetDrainPos()); - } - { - RawBuf buf(5000); - HitRank rank = 10; - buf.addHitRank(rank); - *buf.GetWritableFillPos() = '\0'; - EXPECT_EQUAL(std::string("10"), buf.GetDrainPos()); - } - TEST_DONE(); -} diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index 4877072eacd..13202a062c7 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -48,9 +48,6 @@ class Test : public vespalib::TestApp { void requireThatTermsCanBeEvaluatedInPriorityOrder(); void requireThatBlueprintExposesFieldWithEstimate(); void requireThatBlueprintForcesPositionDataOnChildren(); - void requireThatIteratorHonorsFutureDoom(); - void requireThatIteratorHonorsDoom(); - void requireThatDoomIsPropagated(); public: int Main() override; @@ -79,9 +76,6 @@ Test::Main() TEST_DO(requireThatPhrasesAreUnpacked(true, false, true)); TEST_DO(requireThatBlueprintExposesFieldWithEstimate()); TEST_DO(requireThatBlueprintForcesPositionDataOnChildren()); - TEST_DO(requireThatIteratorHonorsFutureDoom()); - TEST_DO(requireThatIteratorHonorsDoom()); - TEST_DO(requireThatDoomIsPropagated()); TEST_DONE(); } @@ -187,7 +181,7 @@ PhraseSearchTest::PhraseSearchTest(bool expiredDoom) : _requestContext(nullptr, expiredDoom ? vespalib::steady_time(): vespalib::steady_time::max()), _index(), _phrase_fs(field, fieldId, phrase_handle), - _phrase(_phrase_fs, _requestContext, false), + _phrase(_phrase_fs, false), _children(), _md(MatchData::makeTestInstance(100, 10)), _order(), @@ -207,51 +201,6 @@ void Test::requireThatIteratorFindsSimplePhrase(bool useBlueprint) { EXPECT_TRUE(!search->seek(doc_no_match)); } -void Test::requireThatIteratorHonorsFutureDoom() { - PhraseSearchTest test; - test.addTerm("foo", 0).addTerm("bar", 1); - - test.fetchPostings(false); - vespalib::TestClock clock; - vespalib::Doom futureDoom(clock.clock(), vespalib::steady_time::max()); - unique_ptr<SearchIterator> search(test.createSearch(false)); - static_cast<SimplePhraseSearch &>(*search).setDoom(&futureDoom); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_TRUE(search->seek(doc_match)); - EXPECT_TRUE(!search->seek(doc_no_match)); -} - -void Test::requireThatIteratorHonorsDoom() { - PhraseSearchTest test; - test.addTerm("foo", 0).addTerm("bar", 1); - - test.fetchPostings(false); - vespalib::TestClock clock; - vespalib::Doom futureDoom(clock.clock(), vespalib::steady_time()); - unique_ptr<SearchIterator> search(test.createSearch(false)); - static_cast<SimplePhraseSearch &>(*search).setDoom(&futureDoom); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(search->beginId(), search->getDocId()); - EXPECT_TRUE(!search->seek(doc_match)); - EXPECT_TRUE(search->isAtEnd()); - EXPECT_TRUE(!search->seek(doc_no_match)); - EXPECT_TRUE(search->isAtEnd()); -} - -void Test::requireThatDoomIsPropagated() { - PhraseSearchTest test(true); - test.addTerm("foo", 0).addTerm("bar", 1); - - test.fetchPostings(true); - unique_ptr<SearchIterator> search(test.createSearch(true)); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(search->beginId(), search->getDocId()); - EXPECT_TRUE(!search->seek(doc_match)); - EXPECT_TRUE(search->isAtEnd()); - EXPECT_TRUE(!search->seek(doc_no_match)); - EXPECT_TRUE(search->isAtEnd()); -} - void Test::requireThatIteratorFindsLongPhrase(bool useBlueprint) { PhraseSearchTest test; test.addTerm("foo", 0).addTerm("bar", 0).addTerm("baz", 0) @@ -326,9 +275,8 @@ void Test::requireThatTermsCanBeEvaluatedInPriorityOrder() { void Test::requireThatBlueprintExposesFieldWithEstimate() { - FakeRequestContext requestContext; FieldSpec f("foo", 1, 1); - SimplePhraseBlueprint phrase(f, requestContext, false); + SimplePhraseBlueprint phrase(f, false); ASSERT_TRUE(phrase.getState().numFields() == 1); EXPECT_EQUAL(f.getFieldId(), phrase.getState().field(0).getFieldId()); EXPECT_EQUAL(f.getHandle(), phrase.getState().field(0).getHandle()); @@ -352,9 +300,8 @@ Test::requireThatBlueprintExposesFieldWithEstimate() void Test::requireThatBlueprintForcesPositionDataOnChildren() { - FakeRequestContext requestContext; FieldSpec f("foo", 1, 1, true); - SimplePhraseBlueprint phrase(f, requestContext, false); + SimplePhraseBlueprint phrase(f, false); EXPECT_TRUE(f.isFilter()); EXPECT_TRUE(!phrase.getNextChildField(f).isFilter()); } diff --git a/searchlib/src/tests/util/rawbuf_test.cpp b/searchlib/src/tests/util/rawbuf_test.cpp index fd77b5b4ddb..663dde3ef45 100644 --- a/searchlib/src/tests/util/rawbuf_test.cpp +++ b/searchlib/src/tests/util/rawbuf_test.cpp @@ -1,10 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for rawbuf. #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP("rawbuf_test"); @@ -18,40 +16,6 @@ string getString(const RawBuf &buf) { return string(buf.GetDrainPos(), buf.GetUsedLen()); } -TEST("require that rawbuf can append text") { - RawBuf buf(10); - buf += "foo"; - buf += "bar"; - EXPECT_EQUAL("foobar", getString(buf)); -} - -TEST("require that rawbuf expands when appended beyond size") { - RawBuf buf(4); - buf += "foo"; - EXPECT_EQUAL(1u, buf.GetFreeLen()); - buf += "bar"; - EXPECT_EQUAL(2u, buf.GetFreeLen()); - EXPECT_EQUAL("foobar", getString(buf)); -} - -TEST("require that a rawbuf can be appended to another") { - RawBuf buf1(10); - RawBuf buf2(10); - buf1 += "foo"; - buf2 += "bar"; - buf1 += buf2; - EXPECT_EQUAL("foobar", getString(buf1)); -} - -TEST("require that rawbufs can be tested for equality") { - RawBuf buf1(10); - RawBuf buf2(10); - buf1 += "foo"; - buf2 += "bar"; - EXPECT_TRUE(buf1 == buf1); - EXPECT_FALSE(buf1 == buf2); -} - template <typename T> void checkAddNum(void (RawBuf::*addNum)(T, size_t, char), size_t num, size_t fieldw, char fill, const string &expected) { @@ -79,18 +43,6 @@ TEST("require that rawbuf can add numbers in decimal") { checkAddNum(&RawBuf::addNum64, -1, 4, '0', "00-1"); } -TEST("require that rawbuf can add hitrank") { - RawBuf buf(10); - buf.addHitRank(HitRank(4.2)); - EXPECT_EQUAL("4.2", getString(buf)); -} - -TEST("require that rawbuf can add signedhitrank") { - RawBuf buf(10); - buf.addHitRank(SignedHitRank(-4.213)); - EXPECT_EQUAL("-4.213", getString(buf)); -} - TEST("require that rawbuf can append data of known length") { RawBuf buf(10); const string data("foo bar baz qux quux"); @@ -98,50 +50,15 @@ TEST("require that rawbuf can append data of known length") { EXPECT_EQUAL(data, getString(buf)); } -TEST("require that rawbuf can be truncated shorter and longer") { - RawBuf buf(10); - buf += "foobarbaz"; - buf.truncate(3); - buf += "qux"; - buf.truncate(9); - EXPECT_EQUAL("fooquxbaz", getString(buf)); -} - TEST("require that prealloc makes enough room") { RawBuf buf(10); - buf += "foo"; + buf.append("foo"); EXPECT_EQUAL(7u, buf.GetFreeLen()); buf.preAlloc(100); EXPECT_EQUAL("foo", getString(buf)); EXPECT_LESS_EQUAL(100u, buf.GetFreeLen()); } -TEST("require that rawbuf can read from file") { - FastOS_File file("mytemporaryfile"); - ASSERT_TRUE(file.OpenReadWrite()); - ASSERT_EQUAL(6, file.Write2("barbaz", 6)); - file.SetPosition(0); - - RawBuf buf(10); - buf += "foo"; - buf.readFile(file, 3); - EXPECT_EQUAL("foobar", getString(buf)); - buf.readFile(file, 100); - EXPECT_EQUAL("foobarbaz", getString(buf)); - - ASSERT_TRUE(file.Close()); - file.Delete(); -} - -TEST("require that compact discards drained data") { - RawBuf buf(10); - buf += "foobar"; - buf.Drain(3); - buf.Compact(); - buf.Fill(3); - EXPECT_EQUAL("barbar", getString(buf)); -} - TEST("require that reusing a buffer that has grown 4x will alloc new buffer") { RawBuf buf(10); buf.preAlloc(100); @@ -152,7 +69,7 @@ TEST("require that reusing a buffer that has grown 4x will alloc new buffer") { TEST("require that various length and position information can be found.") { RawBuf buf(30); - buf += "foo bar baz qux quux corge"; + buf.append("foo bar baz qux quux corge"); buf.Drain(7); EXPECT_EQUAL(7u, buf.GetDrainLen()); EXPECT_EQUAL(19u, buf.GetUsedLen()); @@ -160,14 +77,6 @@ TEST("require that various length and position information can be found.") { EXPECT_EQUAL(4u, buf.GetFreeLen()); } -TEST("require that rawbuf can 'putToInet' 16-bit numbers") { - RawBuf buf(1); - buf.Put16ToInet(0x1234); - EXPECT_EQUAL(2, buf.GetFillPos() - buf.GetDrainPos()); - EXPECT_EQUAL(0x12, (int) buf.GetDrainPos()[0] & 0xff); - EXPECT_EQUAL(0x34, (int) buf.GetDrainPos()[1] & 0xff); -} - TEST("require that rawbuf can 'putToInet' 32-bit numbers") { RawBuf buf(1); buf.PutToInet(0x12345678); diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h index b32c8fc1663..58aba882e8d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h +++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h @@ -5,7 +5,6 @@ #include "attributememoryfilewriter.h" #include "iattributesavetarget.h" #include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/hash_fun.h> #include <memory> #include <unordered_map> @@ -37,7 +36,7 @@ private: public: AttributeMemorySaveTarget(); - ~AttributeMemorySaveTarget(); + ~AttributeMemorySaveTarget() override; /** * Write the underlying buffer(s) to file(s). diff --git a/searchlib/src/vespa/searchlib/common/hitrank.h b/searchlib/src/vespa/searchlib/common/hitrank.h index 652d431193b..092855878c2 100644 --- a/searchlib/src/vespa/searchlib/common/hitrank.h +++ b/searchlib/src/vespa/searchlib/common/hitrank.h @@ -7,7 +7,6 @@ namespace search { typedef double HitRank; -typedef double SignedHitRank; constexpr HitRank default_rank_value = -HUGE_VAL; constexpr HitRank zero_rank_value = 0.0; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp index 0d0478b2ddc..b67a8409581 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp @@ -20,13 +20,13 @@ ZcRareWordPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features, const PosOccFieldsParams *fieldsParams, - const TermFieldMatchDataArray &matchData) - : ZcRareWordPostingIterator<bigEndian, dynamic_k>(matchData, start, docIdLimit, + TermFieldMatchDataArray matchData) + : ZcRareWordPostingIterator<bigEndian, dynamic_k>(std::move(matchData), start, docIdLimit, decode_normal_features, decode_interleaved_features, unpack_normal_features, unpack_interleaved_features), _decodeContextReal(start.getOccurences(), start.getBitOffset(), bitLength, fieldsParams) { - assert(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); + assert(!this->_matchData.valid() || (fieldsParams->getNumFields() == this->_matchData.size())); _decodeContext = &_decodeContextReal; } @@ -37,19 +37,21 @@ ZcPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit, bool unpack_normal_features, bool unpack_interleaved_features, uint32_t minChunkDocs, const PostingListCounts &counts, const PosOccFieldsParams *fieldsParams, - const TermFieldMatchDataArray &matchData) - : ZcPostingIterator<bigEndian>(minChunkDocs, dynamic_k, counts, matchData, start, docIdLimit, + TermFieldMatchDataArray matchData) + : ZcPostingIterator<bigEndian>(minChunkDocs, dynamic_k, counts, std::move(matchData), start, docIdLimit, decode_normal_features, decode_interleaved_features, unpack_normal_features, unpack_interleaved_features), _decodeContextReal(start.getOccurences(), start.getBitOffset(), bitLength, fieldsParams) { - assert(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); + assert(!this->_matchData.valid() || (fieldsParams->getNumFields() == this->_matchData.size())); _decodeContext = &_decodeContextReal; } template <bool bigEndian> std::unique_ptr<search::queryeval::SearchIterator> -create_zc_posocc_iterator(const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data, bool unpack_normal_features, bool unpack_interleaved_features) +create_zc_posocc_iterator(const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, + const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, + fef::TermFieldMatchDataArray match_data, bool unpack_normal_features, bool unpack_interleaved_features) { using EC = bitcompression::EncodeContext64<bigEndian>; bitcompression::DecodeContext64<bigEndian> d(start.getOccurences(), start.getBitOffset()); @@ -61,28 +63,38 @@ create_zc_posocc_iterator(const PostingListCounts &counts, bitcompression::Posit assert((num_docs == counts._numDocs) || ((num_docs == posting_params._min_chunk_docs) && (num_docs < counts._numDocs))); if (num_docs < posting_params._min_skip_docs) { if (posting_params._dynamic_k) { - return std::make_unique<ZcRareWordPosOccIterator<bigEndian, true>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, unpack_interleaved_features, &fields_params, match_data); + return std::make_unique<ZcRareWordPosOccIterator<bigEndian, true>>(start, bit_length, posting_params._doc_id_limit, + posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, + unpack_interleaved_features, &fields_params, std::move(match_data)); } else { - return std::make_unique<ZcRareWordPosOccIterator<bigEndian, false>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, unpack_interleaved_features, &fields_params, match_data); + return std::make_unique<ZcRareWordPosOccIterator<bigEndian, false>>(start, bit_length, posting_params._doc_id_limit, + posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, + unpack_interleaved_features, &fields_params, std::move(match_data)); } } else { if (posting_params._dynamic_k) { - return std::make_unique<ZcPosOccIterator<bigEndian, true>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, unpack_interleaved_features, posting_params._min_chunk_docs, counts, &fields_params, match_data); + return std::make_unique<ZcPosOccIterator<bigEndian, true>>(start, bit_length, posting_params._doc_id_limit, + posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, + unpack_interleaved_features, posting_params._min_chunk_docs, counts, &fields_params, std::move(match_data)); } else { - return std::make_unique<ZcPosOccIterator<bigEndian, false>>(start, bit_length, posting_params._doc_id_limit, posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, unpack_interleaved_features, posting_params._min_chunk_docs, counts, &fields_params, match_data); + return std::make_unique<ZcPosOccIterator<bigEndian, false>>(start, bit_length, posting_params._doc_id_limit, + posting_params._encode_features, posting_params._encode_interleaved_features, unpack_normal_features, + unpack_interleaved_features, posting_params._min_chunk_docs, counts, &fields_params, std::move(match_data)); } } } std::unique_ptr<search::queryeval::SearchIterator> -create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data) +create_zc_posocc_iterator(bool bigEndian, const PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, + const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, + fef::TermFieldMatchDataArray match_data) { bool unpack_normal_features = match_data.valid() ? match_data[0]->needs_normal_features() : false; bool unpack_interleaved_features = match_data.valid() ? match_data[0]->needs_interleaved_features() : false; if (bigEndian) { - return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_interleaved_features); + return create_zc_posocc_iterator<true>(counts, start, bit_length, posting_params, fields_params, std::move(match_data), unpack_normal_features, unpack_interleaved_features); } else { - return create_zc_posocc_iterator<false>(counts, start, bit_length, posting_params, fields_params, match_data, unpack_normal_features, unpack_interleaved_features); + return create_zc_posocc_iterator<false>(counts, start, bit_length, posting_params, fields_params, std::move(match_data), unpack_normal_features, unpack_interleaved_features); } } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h index fdc54245c1d..7e60c2df112 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h @@ -23,7 +23,7 @@ public: bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features, const bitcompression::PosOccFieldsParams *fieldsParams, - const fef::TermFieldMatchDataArray &matchData); + fef::TermFieldMatchDataArray matchData); }; @@ -42,11 +42,11 @@ public: bool unpack_normal_features, bool unpack_interleaved_features, uint32_t minChunkDocs, const index::PostingListCounts &counts, const bitcompression::PosOccFieldsParams *fieldsParams, - const fef::TermFieldMatchDataArray &matchData); + fef::TermFieldMatchDataArray matchData); }; std::unique_ptr<search::queryeval::SearchIterator> -create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, const fef::TermFieldMatchDataArray &match_data); +create_zc_posocc_iterator(bool bigEndian, const index::PostingListCounts &counts, bitcompression::Position start, uint64_t bit_length, const Zc4PostingParams &posting_params, const bitcompression::PosOccFieldsParams &fields_params, fef::TermFieldMatchDataArray match_data); extern template class ZcRareWordPosOccIterator<false, false>; extern template class ZcRareWordPosOccIterator<false, true>; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp index c7cd9d476a6..6af0ebb6199 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp @@ -16,8 +16,8 @@ using queryeval::RankedSearchIteratorBase; #define DEBUG_ZCPOSTING_PRINTF 0 #define DEBUG_ZCPOSTING_ASSERT 0 -ZcIteratorBase::ZcIteratorBase(const TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit) : - RankedSearchIteratorBase(matchData), +ZcIteratorBase::ZcIteratorBase(TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit) : + RankedSearchIteratorBase(std::move(matchData)), _docIdLimit(docIdLimit), _start(start) { } @@ -37,10 +37,10 @@ ZcIteratorBase::initRange(uint32_t beginid, uint32_t endid) template <bool bigEndian> ZcRareWordPostingIteratorBase<bigEndian>:: -ZcRareWordPostingIteratorBase(const TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, +ZcRareWordPostingIteratorBase(TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features) - : ZcIteratorBase(matchData, start, docIdLimit), + : ZcIteratorBase(std::move(matchData), start, docIdLimit), _decodeContext(nullptr), _residue(0), _prevDocId(0), @@ -56,10 +56,10 @@ ZcRareWordPostingIteratorBase(const TermFieldMatchDataArray &matchData, Position template <bool bigEndian, bool dynamic_k> ZcRareWordPostingIterator<bigEndian, dynamic_k>:: -ZcRareWordPostingIterator(const TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, +ZcRareWordPostingIterator(TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features) - : ZcRareWordPostingIteratorBase<bigEndian>(matchData, start, docIdLimit, + : ZcRareWordPostingIteratorBase<bigEndian>(std::move(matchData), start, docIdLimit, decode_normal_features, decode_interleaved_features, unpack_normal_features, unpack_interleaved_features), _doc_id_k_param() @@ -187,10 +187,10 @@ ZcRareWordPostingIterator<bigEndian, dynamic_k>::readWordStart(uint32_t docIdLim clearUnpacked(); } -ZcPostingIteratorBase::ZcPostingIteratorBase(const TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, +ZcPostingIteratorBase::ZcPostingIteratorBase(TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features) - : ZcIteratorBase(matchData, start, docIdLimit), + : ZcIteratorBase(std::move(matchData), start, docIdLimit), _valI(nullptr), _valIBase(nullptr), _featureSeekPos(0), @@ -216,11 +216,11 @@ ZcPostingIterator<bigEndian>:: ZcPostingIterator(uint32_t minChunkDocs, bool dynamicK, const PostingListCounts &counts, - const search::fef::TermFieldMatchDataArray &matchData, + search::fef::TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features) - : ZcPostingIteratorBase(matchData, start, docIdLimit, + : ZcPostingIteratorBase(std::move(matchData), start, docIdLimit, decode_normal_features, decode_interleaved_features, unpack_normal_features, unpack_interleaved_features), _decodeContext(nullptr), diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h index 07467e28229..36375a25d2b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h @@ -43,7 +43,7 @@ do { \ class ZcIteratorBase : public queryeval::RankedSearchIteratorBase { protected: - ZcIteratorBase(const fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit); + ZcIteratorBase(fef::TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit); virtual void readWordStart(uint32_t docIdLimit) = 0; virtual void rewind(Position start) = 0; void initRange(uint32_t beginid, uint32_t endid) override; @@ -74,7 +74,7 @@ public: uint32_t _field_length; uint32_t _num_occs; - ZcRareWordPostingIteratorBase(const fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, + ZcRareWordPostingIteratorBase(fef::TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features); @@ -127,7 +127,7 @@ class ZcRareWordPostingIterator : public ZcRareWordPostingIteratorBase<bigEndian ZcPostingDocIdKParam<dynamic_k> _doc_id_k_param; public: using ParentClass::_decodeContext; - ZcRareWordPostingIterator(const fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, + ZcRareWordPostingIterator(fef::TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features); void doSeek(uint32_t docId) override; @@ -299,7 +299,7 @@ protected: VESPA_DLL_LOCAL void doL1SkipSeek(uint32_t docId); void doSeek(uint32_t docId) override; public: - ZcPostingIteratorBase(const fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, + ZcPostingIteratorBase(fef::TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features); }; @@ -328,7 +328,7 @@ public: const PostingListCounts &_counts; ZcPostingIterator(uint32_t minChunkDocs, bool dynamicK, const PostingListCounts &counts, - const search::fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit, + search::fef::TermFieldMatchDataArray matchData, Position start, uint32_t docIdLimit, bool decode_normal_features, bool decode_interleaved_features, bool unpack_normal_features, bool unpack_interleaved_features); diff --git a/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp b/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp index bc97e57d0b3..1a868bcb57a 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_rpc_adapter.cpp @@ -8,7 +8,6 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/vespalib/util/compressor.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/searchlib/common/packets.h> diff --git a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt index ba4430ff8e6..398cc0518f8 100644 --- a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt @@ -36,6 +36,7 @@ vespa_add_library(searchlib_fef OBJECT table.cpp tablemanager.cpp termfieldmatchdata.cpp + termfieldmatchdataarray.cpp termfieldmatchdataposition.cpp termmatchdatamerger.cpp utils.cpp diff --git a/searchlib/src/vespa/searchlib/fef/Doxyfile b/searchlib/src/vespa/searchlib/fef/Doxyfile deleted file mode 100644 index 4ca1f146280..00000000000 --- a/searchlib/src/vespa/searchlib/fef/Doxyfile +++ /dev/null @@ -1,1162 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.3.9.1 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = "Feature Execution Framework" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of source -# files, where putting all generated files in the same directory would otherwise -# cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, -# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, -# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, -# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, -# Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is used -# as the annotated text. Otherwise, the brief description is used as-is. If left -# blank, the following values are used ("$name" is automatically replaced with the -# name of the entity): "The $name class" "The $name widget" "The $name file" -# "is" "provides" "specifies" "contains" "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. - -SHOW_DIRECTORIES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = . - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm - -FILE_PATTERNS = - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = NO - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = IAM_DOXYGEN - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superseded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = YES - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes that -# lay further from the root node will be omitted. Note that setting this option to -# 1 or 2 may greatly reduce the computation time needed for large code bases. Also -# note that a graph may be further truncated if the graph's image dimensions are -# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). -# If 0 is used for the depth value (the default), the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index 168f5e4369f..6ede2eca73c 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -317,12 +317,10 @@ IgnoreDefaultFeatures::check(const Properties &props) namespace matching { const vespalib::string SplitUnpackingIterators::NAME("vespa.matching.split_unpacking_iterators"); -const bool SplitUnpackingIterators::DEFAULT_VALUE(false); -bool SplitUnpackingIterators::check(const Properties &props) { return lookupBool(props, NAME, DEFAULT_VALUE); } - -const vespalib::string DelayUnpackingIterators::NAME("vespa.matching.delay_unpacking_iterators"); -const bool DelayUnpackingIterators::DEFAULT_VALUE(false); -bool DelayUnpackingIterators::check(const Properties &props) { return lookupBool(props, NAME, DEFAULT_VALUE); } +const bool SplitUnpackingIterators::DEFAULT_VALUE(true); +bool SplitUnpackingIterators::check(const Properties &props, bool fallback) { + return lookupBool(props, NAME, fallback); +} const vespalib::string TermwiseLimit::NAME("vespa.matching.termwise_limit"); const double TermwiseLimit::DEFAULT_VALUE(1.0); diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index ce3798a6c8e..fb6c0e5560b 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -237,18 +237,8 @@ namespace matching { struct SplitUnpackingIterators { static const vespalib::string NAME; static const bool DEFAULT_VALUE; - static bool check(const Properties &props); - }; - - /** - * When enabled, iterators that unpack posting information as part - * of matching will be tagged as expensive, in order to delay - * their execution within the iterator tree. - **/ - struct DelayUnpackingIterators { - static const vespalib::string NAME; - static const bool DEFAULT_VALUE; - static bool check(const Properties &props); + static bool check(const Properties &props) { return check(props, DEFAULT_VALUE); } + static bool check(const Properties &props, bool fallback); }; /** diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 30e220b0a34..f725756c269 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -37,7 +37,6 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _secondPhaseRankFeature(), _degradationAttribute(), _split_unpacking_iterators(false), - _delay_unpacking_iterators(false), _termwise_limit(1.0), _numThreads(0), _minHitsPerThread(0), @@ -98,7 +97,6 @@ RankSetup::configure() _feature_rename_map[rename.first] = rename.second; } split_unpacking_iterators(matching::SplitUnpackingIterators::check(_indexEnv.getProperties())); - delay_unpacking_iterators(matching::DelayUnpackingIterators::check(_indexEnv.getProperties())); set_termwise_limit(matching::TermwiseLimit::lookup(_indexEnv.getProperties())); setNumThreadsPerSearch(matching::NumThreadsPerSearch::lookup(_indexEnv.getProperties())); setMinHitsPerThread(matching::MinHitsPerThread::lookup(_indexEnv.getProperties())); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index 5b39715d5a1..866bf2286db 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -47,7 +47,6 @@ private: vespalib::string _secondPhaseRankFeature; vespalib::string _degradationAttribute; bool _split_unpacking_iterators; - bool _delay_unpacking_iterators; double _termwise_limit; uint32_t _numThreads; uint32_t _minHitsPerThread; @@ -144,9 +143,6 @@ public: bool split_unpacking_iterators() const { return _split_unpacking_iterators; } void split_unpacking_iterators(bool value) { _split_unpacking_iterators = value; } - bool delay_unpacking_iterators() const { return _delay_unpacking_iterators; } - void delay_unpacking_iterators(bool value) { _delay_unpacking_iterators = value; } - /** * Set the termwise limit * diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.cpp new file mode 100644 index 00000000000..87a25a87a80 --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.cpp @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "termfieldmatchdataarray.h" + +namespace search::fef { + +TermFieldMatchDataArray::~TermFieldMatchDataArray() = default; + +} diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h index 13a78a445e8..3c1b76ad40e 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h @@ -23,6 +23,12 @@ private: std::vector<TermFieldMatchData *> _array; public: + TermFieldMatchDataArray() = default; + TermFieldMatchDataArray(TermFieldMatchDataArray &&) noexcept = default; + TermFieldMatchDataArray & operator = (TermFieldMatchDataArray &&) noexcept = default; + TermFieldMatchDataArray(const TermFieldMatchDataArray&) = default; + TermFieldMatchDataArray & operator = (const TermFieldMatchDataArray &) = delete; + ~TermFieldMatchDataArray(); /** * Reserve space for a number of elements in order to reduce number of allocations. * @param size Number of elements to reserve space for. @@ -37,7 +43,7 @@ public: * @param value the pointer to be added **/ TermFieldMatchDataArray &add(TermFieldMatchData *value) { - assert(value != 0); + assert(value != nullptr); _array.push_back(value); return *this; } diff --git a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp index 6b9c5a31c97..d1906f53514 100644 --- a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp +++ b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp @@ -6,9 +6,9 @@ namespace search::fef { TermMatchDataMerger::TermMatchDataMerger(const Inputs &allinputs, - const TermFieldMatchDataArray &outputs) + TermFieldMatchDataArray outputs) : _inputs(), - _output(outputs), + _output(std::move(outputs)), _scratch() { for (size_t i = 0; i < _output.size(); ++i) { diff --git a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h index 576dfc9a22c..fd015a7d304 100644 --- a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h +++ b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.h @@ -31,7 +31,7 @@ public: TermMatchDataMerger(const TermMatchDataMerger &) = delete; TermMatchDataMerger &operator=(const TermMatchDataMerger &) = delete; - TermMatchDataMerger(const Inputs &allinputs, const TermFieldMatchDataArray &outputs); + TermMatchDataMerger(const Inputs &allinputs, TermFieldMatchDataArray outputs); ~TermMatchDataMerger(); void merge(uint32_t docid); diff --git a/searchlib/src/vespa/searchlib/index/docbuilder.cpp b/searchlib/src/vespa/searchlib/index/docbuilder.cpp index d4bf9e86c74..bf0a6866a1a 100644 --- a/searchlib/src/vespa/searchlib/index/docbuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/docbuilder.cpp @@ -111,21 +111,6 @@ insertRaw(const Schema::Field & sfield, rfvalue->setValue(static_cast<const char *>(buf), len); } - -template <typename T> -std::unique_ptr<T> -make_UP(T *p) -{ - return std::unique_ptr<T>(p); -} - -template <typename T> -std::unique_ptr<T> -makeUP(T *p) -{ - return std::unique_ptr<T>(p); -} - } namespace docbuilderkludge @@ -154,10 +139,10 @@ using namespace docbuilderkludge; namespace { -std::unique_ptr<Annotation> +Annotation makeTokenType(linguistics::TokenType type) { - return std::make_unique<Annotation>(*AnnotationType::TOKEN_TYPE, std::make_unique<IntFieldValue>(type)); + return Annotation(*AnnotationType::TOKEN_TYPE, std::make_unique<IntFieldValue>(type)); } } @@ -337,7 +322,7 @@ DocBuilder::IndexFieldHandle::addTokenizedString(const vespalib::string &val, void DocBuilder::IndexFieldHandle::addSpan(size_t start, size_t len) { - const SpanNode &span = _spanList->add(makeUP(new Span(start, len))); + const SpanNode &span = _spanList->add(std::make_unique<Span>(start, len)); _lastSpan = &span; } @@ -388,8 +373,8 @@ DocBuilder::IndexFieldHandle::addTermAnnotation(const vespalib::string &val) assert(_spanTree); assert(_lastSpan != nullptr); _spanTree->annotate(*_lastSpan, - makeUP(new Annotation(*AnnotationType::TERM, - makeUP(new StringFieldValue(val))))); + Annotation(*AnnotationType::TERM, + std::make_unique<StringFieldValue>(val))); } void diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp b/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp index ba1dcb9c87a..714331bc161 100644 --- a/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp @@ -10,7 +10,6 @@ PostingListHandle::createIterator(const PostingListCounts &counts, const search::fef::TermFieldMatchDataArray &matchData, bool useBitVector) const { - (void) useBitVector; return _file->createIterator(counts, *this, matchData, useBitVector); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 59289bfbf8f..ddf58d7b9f1 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -215,10 +215,10 @@ template <bool interleaved_features> queryeval::SearchIterator::UP FieldIndex<interleaved_features>::make_search_iterator(const vespalib::string& term, uint32_t field_id, - const fef::TermFieldMatchDataArray& match_data) const + fef::TermFieldMatchDataArray match_data) const { return search::memoryindex::make_search_iterator<interleaved_features> - (find(term), getFeatureStore(), field_id, match_data); + (find(term), getFeatureStore(), field_id, std::move(match_data)); } namespace { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index 988b7d723f1..9f9f4124100 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -100,7 +100,7 @@ public: */ queryeval::SearchIterator::UP make_search_iterator(const vespalib::string& term, uint32_t field_id, - const fef::TermFieldMatchDataArray& match_data) const; + fef::TermFieldMatchDataArray match_data) const; std::unique_ptr<queryeval::SimpleLeafBlueprint> make_term_blueprint(const vespalib::string& term, const queryeval::FieldSpecBase& field, diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index 2b4c1a024d9..48fc6873390 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -33,7 +33,7 @@ public: PostingIteratorBase(PostingListIteratorType itr, const FeatureStore& feature_store, uint32_t field_id, - const fef::TermFieldMatchDataArray& match_data); + fef::TermFieldMatchDataArray match_data); ~PostingIteratorBase(); void doSeek(uint32_t docId) override; @@ -45,8 +45,8 @@ template <bool interleaved_features> PostingIteratorBase<interleaved_features>::PostingIteratorBase(PostingListIteratorType itr, const FeatureStore& feature_store, uint32_t field_id, - const fef::TermFieldMatchDataArray& match_data) : - queryeval::RankedSearchIteratorBase(match_data), + fef::TermFieldMatchDataArray match_data) : + queryeval::RankedSearchIteratorBase(std::move(match_data)), _itr(itr), _feature_store(feature_store), _feature_decoder(nullptr) @@ -141,25 +141,25 @@ queryeval::SearchIterator::UP make_search_iterator(typename FieldIndex<interleaved_features>::PostingList::ConstIterator itr, const FeatureStore& feature_store, uint32_t field_id, - const fef::TermFieldMatchDataArray& match_data) + fef::TermFieldMatchDataArray match_data) { assert(match_data.size() == 1); auto* tfmd = match_data[0]; if (tfmd->needs_normal_features()) { if (tfmd->needs_interleaved_features()) { return std::make_unique<PostingIterator<interleaved_features, true, true>> - (itr, feature_store, field_id, match_data); + (itr, feature_store, field_id, std::move(match_data)); } else { return std::make_unique<PostingIterator<interleaved_features, true, false>> - (itr, feature_store, field_id, match_data); + (itr, feature_store, field_id, std::move(match_data)); } } else { if (tfmd->needs_interleaved_features()) { return std::make_unique<PostingIterator<interleaved_features, false, true>> - (itr, feature_store, field_id, match_data); + (itr, feature_store, field_id, std::move(match_data)); } else { return std::make_unique<PostingIterator<interleaved_features, false, false>> - (itr, feature_store, field_id, match_data); + (itr, feature_store, field_id, std::move(match_data)); } } } @@ -169,14 +169,14 @@ queryeval::SearchIterator::UP make_search_iterator<false>(typename FieldIndex<false>::PostingList::ConstIterator, const FeatureStore&, uint32_t, - const fef::TermFieldMatchDataArray&); + fef::TermFieldMatchDataArray); template queryeval::SearchIterator::UP make_search_iterator<true>(typename FieldIndex<true>::PostingList::ConstIterator, const FeatureStore&, uint32_t, - const fef::TermFieldMatchDataArray&); + fef::TermFieldMatchDataArray); template class PostingIteratorBase<false>; template class PostingIteratorBase<true>; diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h index ea513b1dced..790f8bb3db7 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h @@ -22,7 +22,7 @@ queryeval::SearchIterator::UP make_search_iterator(typename FieldIndex<interleaved_features>::PostingList::ConstIterator itr, const FeatureStore& feature_store, uint32_t field_id, - const fef::TermFieldMatchDataArray& match_data); + fef::TermFieldMatchDataArray match_data); } diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index c3b5fcc81fa..1285125d34b 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchlib/query/weight.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 5ed33ec18a1..9bb62e76c49 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/searchlib/parsequery/parse.h> +#include <vespa/searchlib/util/rawbuf.h> using vespalib::string; using std::vector; diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearch.h b/searchlib/src/vespa/searchlib/queryeval/andsearch.h index 85df54c81d8..a3e7c074cf1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearch.h @@ -23,7 +23,7 @@ public: AndSearch & estimate(uint32_t est) { _estimate = est; return *this; } uint32_t estimate() const { return _estimate; } protected: - AndSearch(Children children); + explicit AndSearch(Children children); void doUnpack(uint32_t docid) override; UP andWith(UP filter, uint32_t estimate) override; UP offerFilterToChildren(UP filter, uint32_t estimate); diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 4d7460452f2..5dca5c9412b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -362,8 +362,7 @@ public: void freeze() override final; SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const override; - virtual SearchIteratorUP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, - bool strict) const = 0; + virtual SearchIteratorUP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const = 0; }; // for leaf nodes representing a single term diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp index 5b8757411bd..d179515be6c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp @@ -31,7 +31,7 @@ CreateBlueprintVisitorHelper::getResult() void CreateBlueprintVisitorHelper::visitPhrase(query::Phrase &n) { - auto phrase = std::make_unique<SimplePhraseBlueprint>(_field, _requestContext, n.is_expensive()); + auto phrase = std::make_unique<SimplePhraseBlueprint>(_field, n.is_expensive()); for (const query::Node * child : n.getChildren()) { FieldSpecList fields; fields.add(phrase->getNextChildField(_field)); diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h index 9ad2e4dc92d..5955c359003 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h @@ -21,7 +21,7 @@ class DotProductBlueprint : public ComplexLeafBlueprint public: DotProductBlueprint(const FieldSpec &field); - ~DotProductBlueprint(); + ~DotProductBlueprint() override; // used by create visitor FieldSpec getNextChildField(const FieldSpec &outer); diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.h b/searchlib/src/vespa/searchlib/queryeval/fake_search.h index d8cd31fba4c..5cd04f80499 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.h @@ -29,9 +29,9 @@ public: const vespalib::string &field, const vespalib::string &term, const FakeResult &res, - const fef::TermFieldMatchDataArray &tfmda) + fef::TermFieldMatchDataArray tfmda) : _tag(tag), _field(field), _term(term), - _result(res), _offset(0), _tfmda(tfmda), + _result(res), _offset(0), _tfmda(std::move(tfmda)), _ctx(nullptr) { assert(_tfmda.size() == 1); diff --git a/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h b/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h index 5aa5db081ab..52992a52103 100644 --- a/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h +++ b/searchlib/src/vespa/searchlib/queryeval/irequestcontext.h @@ -17,7 +17,7 @@ namespace search::queryeval { class IRequestContext { public: - virtual ~IRequestContext() { } + virtual ~IRequestContext() = default; /** * Provides the time of soft doom for the query. Now it is time to start cleaning up and return what you have. diff --git a/searchlib/src/vespa/searchlib/queryeval/iterators.cpp b/searchlib/src/vespa/searchlib/queryeval/iterators.cpp index f3d12cd34a6..07beebac695 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterators.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/iterators.cpp @@ -5,9 +5,9 @@ namespace search::queryeval { RankedSearchIteratorBase:: -RankedSearchIteratorBase(const fef::TermFieldMatchDataArray &matchData) +RankedSearchIteratorBase(fef::TermFieldMatchDataArray matchData) : SearchIterator(), - _matchData(matchData), + _matchData(std::move(matchData)), _needUnpack(1) { } diff --git a/searchlib/src/vespa/searchlib/queryeval/iterators.h b/searchlib/src/vespa/searchlib/queryeval/iterators.h index ead00437471..e4f75184924 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterators.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterators.h @@ -21,7 +21,7 @@ protected: uint32_t getNeedUnpack() const { return _needUnpack; } void incNeedUnpack() { ++_needUnpack; } public: - RankedSearchIteratorBase(const fef::TermFieldMatchDataArray &matchData); + RankedSearchIteratorBase(fef::TermFieldMatchDataArray matchData); ~RankedSearchIteratorBase() override; bool getUnpacked() const { return _needUnpack == 0; } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h index 7a1d8f3d253..2f97d89e322 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h @@ -44,7 +44,7 @@ protected: createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, bool strict) const override; public: SimpleBlueprint(const SimpleResult &result); - ~SimpleBlueprint(); + ~SimpleBlueprint() override; SimpleBlueprint &tag(const vespalib::string &tag); const vespalib::string &tag() const { return _tag; } SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -67,7 +67,7 @@ protected: public: FakeBlueprint(const FieldSpec &field, const FakeResult &result); - ~FakeBlueprint(); + ~FakeBlueprint() override; FakeBlueprint &tag(const vespalib::string &t) { _tag = t; diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp index 5c932b3aeb8..16f4012f0e7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp @@ -42,9 +42,8 @@ MultiSearch::MultiSearch(Children children) { } -MultiSearch::~MultiSearch() -{ -} +MultiSearch::MultiSearch() = default; +MultiSearch::~MultiSearch() = default; void MultiSearch::initRange(uint32_t beginid, uint32_t endid) diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.h b/searchlib/src/vespa/searchlib/queryeval/multisearch.h index 73c31d243db..9216391b85d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.h @@ -32,8 +32,8 @@ public: * @param children the search objects we are and'ing * this object takes ownership of the children. **/ - MultiSearch(Children children); - virtual ~MultiSearch() override; + explicit MultiSearch(Children children); + ~MultiSearch() override; const Children & getChildren() const { return _children; } virtual bool isAnd() const { return false; } virtual bool isAndNot() const { return false; } @@ -42,7 +42,7 @@ public: virtual bool needUnpack(size_t index) const { (void) index; return true; } void initRange(uint32_t beginId, uint32_t endId) override; protected: - MultiSearch() {} + MultiSearch(); void doUnpack(uint32_t docid) override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; private: diff --git a/searchlib/src/vespa/searchlib/queryeval/searchable.h b/searchlib/src/vespa/searchlib/queryeval/searchable.h index 6202f5d1f0d..2467cfe4142 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchable.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchable.h @@ -36,7 +36,7 @@ protected: public: typedef std::shared_ptr<Searchable> SP; - Searchable() {} + Searchable() = default; /** * Create a blueprint searching a set of fields. The default @@ -51,7 +51,7 @@ public: virtual Blueprint::UP createBlueprint(const IRequestContext & requestContext, const FieldSpecList &fields, const search::query::Node &term); - virtual ~Searchable() {} + virtual ~Searchable() = default; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp index 0025fd5fe03..2ef7a2cf0a0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp @@ -11,9 +11,8 @@ namespace search::queryeval { -SimplePhraseBlueprint::SimplePhraseBlueprint(const FieldSpec &field, const IRequestContext & requestContext, bool expensive) +SimplePhraseBlueprint::SimplePhraseBlueprint(const FieldSpec &field, bool expensive) : ComplexLeafBlueprint(field), - _doom(requestContext.getDoom()), _field(field), _estimate(), _layout(), @@ -75,15 +74,14 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd order_map.insert(std::make_pair(childState.estimate().estHits, i)); } std::vector<uint32_t> eval_order; + eval_order.reserve(order_map.size()); for (const auto & child : order_map) { eval_order.push_back(child.second); - } - - auto phrase = std::make_unique<SimplePhraseSearch>(std::move(children), - std::move(md), childMatch, - eval_order, *tfmda[0], strict); - phrase->setDoom(& _doom); - return phrase; + } + + return std::make_unique<SimplePhraseSearch>(std::move(children), + std::move(md), std::move(childMatch), + std::move(eval_order), *tfmda[0], strict); } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h index 10cdac34f19..5ae7673269f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h @@ -5,7 +5,6 @@ #include "searchable.h" #include "irequestcontext.h" #include <vespa/searchlib/fef/matchdatalayout.h> -#include <vespa/vespalib/util/doom.h> namespace search::fef { class TermFieldMatchData; } @@ -14,18 +13,17 @@ namespace search::queryeval { class SimplePhraseBlueprint : public ComplexLeafBlueprint { private: - const vespalib::Doom _doom; FieldSpec _field; HitEstimate _estimate; fef::MatchDataLayout _layout; std::vector<Blueprint*> _terms; - SimplePhraseBlueprint(const SimplePhraseBlueprint &); // disabled - SimplePhraseBlueprint &operator=(const SimplePhraseBlueprint &); // disabled - public: - SimplePhraseBlueprint(const FieldSpec &field, const IRequestContext & requestContext, bool expensive); - ~SimplePhraseBlueprint(); + SimplePhraseBlueprint(const FieldSpec &field, bool expensive); + SimplePhraseBlueprint(const SimplePhraseBlueprint &) = delete; + SimplePhraseBlueprint &operator=(const SimplePhraseBlueprint &) = delete; + + ~SimplePhraseBlueprint() override; // used by create visitor FieldSpec getNextChildField(const FieldSpec &outer); diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp index f58f888393b..f5069fd4f53 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp @@ -22,17 +22,21 @@ class PhraseMatcher { uint32_t _element_id; uint32_t _position; - TermFieldMatchData::PositionsIterator &iterator(uint32_t word_index) - { return _iterators[word_index]; } + TermFieldMatchData::PositionsIterator &iterator(uint32_t word_index) { + return _iterators[word_index]; + } - TermFieldMatchData::PositionsIterator end(uint32_t word_index) - { return _tmds[word_index]->end(); } + TermFieldMatchData::PositionsIterator end(uint32_t word_index) { + return _tmds[word_index]->end(); + } - uint32_t elementId(uint32_t word_index) - { return iterator(word_index)->getElementId(); } + uint32_t elementId(uint32_t word_index) { + return iterator(word_index)->getElementId(); + } - uint32_t position(uint32_t word_index) - { return iterator(word_index)->getPosition(); } + uint32_t position(uint32_t word_index) { + return iterator(word_index)->getPosition(); + } void iterateToElement(uint32_t word_index) { while (iterator(word_index) != end(word_index) && @@ -145,13 +149,9 @@ allTermsHaveMatch(const SimplePhraseSearch::Children &terms, const vector<uint32 void SimplePhraseSearch::phraseSeek(uint32_t doc_id) { if (allTermsHaveMatch(getChildren(), _eval_order, doc_id)) { - if (doom()) { - setAtEnd(); - } else { - AndSearch::doUnpack(doc_id); - if (PhraseMatcher(_childMatch, _eval_order, _iterators).hasMatch()) { - setDocId(doc_id); - } + AndSearch::doUnpack(doc_id); + if (PhraseMatcher(_childMatch, _eval_order, _iterators).hasMatch()) { + setDocId(doc_id); } } } @@ -159,19 +159,18 @@ SimplePhraseSearch::phraseSeek(uint32_t doc_id) { SimplePhraseSearch::SimplePhraseSearch(Children children, fef::MatchData::UP md, - const fef::TermFieldMatchDataArray &childMatch, + fef::TermFieldMatchDataArray childMatch, vector<uint32_t> eval_order, TermFieldMatchData &tmd, bool strict) : AndSearch(std::move(children)), _md(std::move(md)), - _childMatch(childMatch), + _childMatch(std::move(childMatch)), _eval_order(std::move(eval_order)), _tmd(tmd), - _doom(nullptr), _strict(strict), _iterators(getChildren().size()) { - assert(getChildren().size() > 0); + assert( ! getChildren().empty()); assert(getChildren().size() == _childMatch.size()); assert(getChildren().size() == _eval_order.size()); } diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h index 7b9d7c9365f..5b0c1401c85 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h @@ -7,7 +7,6 @@ #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/vespalib/util/doom.h> #include <memory> #include <vector> @@ -22,7 +21,6 @@ class SimplePhraseSearch : public AndSearch fef::TermFieldMatchDataArray _childMatch; std::vector<uint32_t> _eval_order; fef::TermFieldMatchData &_tmd; - const vespalib::Doom *_doom; bool _strict; typedef fef::TermFieldMatchData::PositionsIterator It; @@ -30,8 +28,6 @@ class SimplePhraseSearch : public AndSearch std::vector<It> _iterators; void phraseSeek(uint32_t doc_id); - bool doom() const { return ((_doom != nullptr) && _doom->soft_doom()); } - public: /** * Takes ownership of the contents of children. @@ -45,13 +41,12 @@ public: **/ SimplePhraseSearch(Children children, fef::MatchData::UP md, - const fef::TermFieldMatchDataArray &childMatch, + fef::TermFieldMatchDataArray childMatch, std::vector<uint32_t> eval_order, fef::TermFieldMatchData &tmd, bool strict); void doSeek(uint32_t doc_id) override; void doUnpack(uint32_t doc_id) override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; - SimplePhraseSearch & setDoom(const vespalib::Doom * doom) { _doom = doom; return *this; } }; } diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.cpp b/searchlib/src/vespa/searchlib/util/rawbuf.cpp index d3cc9996c34..2ce9d52b2ce 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.cpp +++ b/searchlib/src/vespa/searchlib/util/rawbuf.cpp @@ -2,14 +2,12 @@ #include "rawbuf.h" #include <vespa/vespalib/util/compress.h> -#include <vespa/fastos/file.h> #include <cassert> #include <cstring> +#include <cstdlib> namespace search { -static inline size_t smin(size_t a, size_t b) { return (a < b) ? a : b; } - RawBuf::RawBuf(size_t size) : _bufStart(nullptr), _bufEnd(nullptr), @@ -25,21 +23,6 @@ RawBuf::RawBuf(size_t size) _bufDrainPos = _bufFillPos = _bufStart; } - -RawBuf::RawBuf(char *start, size_t size) - : _bufStart(nullptr), - _bufEnd(nullptr), - _bufFillPos(nullptr), - _bufDrainPos(nullptr), - _initialBufStart(start), - _initialSize(size) -{ - _bufStart = start; - _bufEnd = _bufStart + size; - _bufDrainPos = _bufFillPos = _bufStart; -} - - RawBuf::~RawBuf() { if (_bufStart != _initialBufStart) @@ -109,17 +92,6 @@ RawBuf::appendCompressedNumber(int64_t n) _bufFillPos += vespalib::compress::Integer::compress(n, _bufFillPos); } - -/** - * Has the entire contents of the buffer been used up, i.e. freed? - */ -bool -RawBuf::IsEmpty() -{ - return _bufFillPos == _bufDrainPos; -} - - /** * Free 'len' bytes from the start of the contents. (These * have presumably been written or read.) @@ -158,19 +130,6 @@ RawBuf::preAlloc(size_t len) assert(static_cast<size_t>(_bufEnd -_bufFillPos) >= len); } - -void -RawBuf::Compact() -{ - if (_bufDrainPos == _bufStart) - return; - if (_bufFillPos != _bufDrainPos) - memmove(_bufStart, _bufDrainPos, _bufFillPos - _bufDrainPos); - _bufFillPos -= (_bufDrainPos - _bufStart); - _bufDrainPos = _bufStart; -} - - void RawBuf::Reuse() { @@ -191,7 +150,7 @@ RawBuf::Reuse() void -RawBuf::operator+=(const char *src) +RawBuf::append(const char *src) { while (*src) { char *cachedBufFillPos = _bufFillPos; @@ -204,37 +163,6 @@ RawBuf::operator+=(const char *src) } } - -void -RawBuf::operator+=(const RawBuf& buffer) -{ - size_t nbytes = buffer.GetUsedLen(); - if (nbytes == 0) - return; - - while (GetFreeLen() < nbytes) - expandBuf(nbytes); - memcpy(_bufFillPos, buffer._bufDrainPos, nbytes); - _bufFillPos += nbytes; -} - - -bool -RawBuf::operator==(const RawBuf &buffer) const -{ - size_t nbytes = buffer.GetUsedLen(); - if (nbytes != GetUsedLen()) - return false; - - const char *p, *t; - for (p=_bufDrainPos, t=buffer._bufDrainPos; p<_bufFillPos; p++, t++) { - if (*p != *t) - return false; - } - - return true; -} - /** * Append the value of param 'num' to the buffer, as a decimal * number right adjusted in a field of width 'fieldw', remaining @@ -327,37 +255,6 @@ RawBuf::addNum64(int64_t num, size_t fieldw, char fill) _bufFillPos = cachedBufFillPos; } - -void -RawBuf::addHitRank(HitRank num) -{ - char buf1[100]; - snprintf(buf1, sizeof(buf1), "%g", static_cast<double>(num)); - append(buf1, strlen(buf1)); -} - - -void -RawBuf::addSignedHitRank(SignedHitRank num) -{ - char buf1[100]; - snprintf(buf1, sizeof(buf1), "%g", static_cast<double>(num)); - append(buf1, strlen(buf1)); -} - -/** - * Read from the indicated file into the buffer, no more that the - * given number of bytes and no more than will fit in the buffer. - */ -size_t -RawBuf::readFile(FastOS_FileInterface &file, size_t maxlen) -{ - size_t got = file.Read(_bufFillPos, smin((_bufEnd - _bufFillPos), maxlen)); - if (got > 0) - _bufFillPos += got; - return got; -} - void RawBuf::ensureSizeInternal(size_t size) { expandBuf(size); diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.h b/searchlib/src/vespa/searchlib/util/rawbuf.h index e69c13e13b3..a79e35be6c7 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.h +++ b/searchlib/src/vespa/searchlib/util/rawbuf.h @@ -2,11 +2,8 @@ #pragma once -#include <vespa/searchlib/common/hitrank.h> #include <cstdint> -#include <sys/types.h> - -class FastOS_FileInterface; +#include <cstddef> namespace search { /** @@ -19,9 +16,6 @@ namespace search { class RawBuf { private: - RawBuf(const RawBuf &); - RawBuf& operator=(const RawBuf &); - char* _bufStart; // ref. to start of buffer (don't move this!) char* _bufEnd; // ref. to byte after last in buffer (don't mo) char* _bufFillPos; // ref. to byte where next should be put in @@ -30,41 +24,48 @@ private: size_t _initialSize; void ensureSizeInternal(size_t size); + void expandBuf(size_t needlen); + /** + * Convert unsigned int.s 'src', to interNet highendian order, at 'dst' + * or _bufFillPos. Update or return ref to next char after those filled in. + */ + static unsigned char* ToInet(uint32_t src, unsigned char* dst) { + *(dst + 3) = src; // The least significant 8 bits + src >>= 8; // of 'src' are stored. + *(dst + 2) = src; + src >>= 8; + *(dst + 1) = src; + src >>= 8; + *dst = src; + return dst + 4; + }; + static unsigned char* ToInet(uint64_t src, unsigned char* dst) { + ToInet(static_cast<uint32_t>(src >> 32), dst); + ToInet(static_cast<uint32_t>(src & 0xffffffffull), dst + 4); + return dst + 8; + }; public: - - RawBuf(char *start, size_t size);// Initially use provided buffer - RawBuf(size_t size); // malloc-s given size, assigns to _bufStart + RawBuf(const RawBuf &) = delete; + RawBuf& operator=(const RawBuf &) = delete; + explicit RawBuf(size_t size); // malloc-s given size, assigns to _bufStart ~RawBuf(); // Frees _bufStart, i.e. the char[]. - void operator+=(const char *src); - void operator+=(const RawBuf& buffer); - bool operator==(const RawBuf &buffer) const; void addNum(size_t num, size_t fieldw, char fill); void addNum32(int32_t num, size_t fieldw, char fill); void addNum64(int64_t num, size_t fieldw, char fill); - void addHitRank(HitRank num); - void addSignedHitRank(SignedHitRank num); - void append(const void *data, size_t len); + void append(const char *data); void append(uint8_t byte); - void appendLong(uint64_t n); void appendCompressedPositiveNumber(uint64_t n); void appendCompressedNumber(int64_t n); - bool IsEmpty(); // Return whether all written. - void expandBuf(size_t needlen); size_t GetFreeLen() const { return _bufEnd - _bufFillPos; } size_t GetDrainLen() const { return _bufDrainPos - _bufStart; } const char *GetDrainPos() const { return _bufDrainPos; } const char *GetFillPos() const { return _bufFillPos; } - char * GetWritableFillPos() const { return _bufFillPos; } char * GetWritableFillPos(size_t len) { preAlloc(len); return _bufFillPos; } - char * GetWritableDrainPos(size_t offset) { return _bufDrainPos + offset; } - void truncate(size_t offset) { _bufFillPos = _bufDrainPos + offset; } void preAlloc(size_t len); // Ensure room for 'len' more bytes. - size_t readFile(FastOS_FileInterface &file, size_t maxlen); void reset() { _bufDrainPos = _bufFillPos = _bufStart; } - void Compact(); void Reuse(); size_t GetUsedAndDrainLen() const { return _bufFillPos - _bufStart; } size_t GetUsedLen() const { return _bufFillPos - _bufDrainPos; } @@ -77,68 +78,17 @@ public: } } - /** - * Convert from interNet highendian order at 'src', to unsigned integers - */ - static uint16_t InetTo16(const unsigned char *src) { - return (static_cast<uint16_t>(*src) << 8) + *(src + 1); - }; - static uint16_t InetTo16(const char* src) { - return InetTo16(reinterpret_cast<const unsigned char *>(src)); - }; - static uint32_t InetTo32(const unsigned char* src) { - return (((((static_cast<uint32_t>(*src) << 8) + *(src + 1)) << 8) - + *(src + 2)) << 8) + *(src + 3); - }; - static uint32_t InetTo32(const char* src) { - return InetTo32(reinterpret_cast<const unsigned char *>(src)); - }; - - /** - * Convert unsigned int.s 'src', to interNet highendian order, at 'dst' - * or _bufFillPos. Update or return ref to next char after those filled in. - */ - static unsigned char* ToInet(uint16_t src, unsigned char* dst) { - *(dst + 1) = static_cast<unsigned char>(src); // The least significant 8 bits - src >>= 8; // of 'src' are stored. - *dst = static_cast<unsigned char>(src); - return dst + 2; - }; - void Put16ToInet(uint16_t src) { - ensureSize(2); - _bufFillPos = reinterpret_cast<char *> - (ToInet(src, - reinterpret_cast<unsigned char*>(_bufFillPos))); - }; - static unsigned char* ToInet(uint32_t src, unsigned char* dst) { - *(dst + 3) = src; // The least significant 8 bits - src >>= 8; // of 'src' are stored. - *(dst + 2) = src; - src >>= 8; - *(dst + 1) = src; - src >>= 8; - *dst = src; - return dst + 4; - }; void PutToInet(uint32_t src) { ensureSize(4); - _bufFillPos = reinterpret_cast<char *> - (ToInet(src, - reinterpret_cast<unsigned char*>(_bufFillPos))); + _bufFillPos = reinterpret_cast<char *>(ToInet(src,reinterpret_cast<unsigned char*>(_bufFillPos))); }; - static unsigned char* ToInet(uint64_t src, unsigned char* dst) { - ToInet(static_cast<uint32_t>(src >> 32), dst); - ToInet(static_cast<uint32_t>(src & 0xffffffffull), dst + 4); - return dst + 8; - }; void Put64ToInet(uint64_t src) { ensureSize(8); - _bufFillPos = reinterpret_cast<char *> - (ToInet(src, - reinterpret_cast<unsigned char*>(_bufFillPos))); + _bufFillPos = reinterpret_cast<char *>(ToInet(src,reinterpret_cast<unsigned char*>(_bufFillPos))); }; + }; } diff --git a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h index 5cdfec78ec1..d00a0714045 100644 --- a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h +++ b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h @@ -2,12 +2,12 @@ #pragma once -#include <vespa/vespalib/data/output.h> #include "rawbuf.h" +#include <vespa/vespalib/data/output.h> namespace search { -class SlimeOutputRawBufAdapter : public ::vespalib::Output +class SlimeOutputRawBufAdapter : public vespalib::Output { private: RawBuf &_buf; diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 60f85c07b86..d36bfdc0bae 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -22,6 +22,5 @@ vespa_define_module( src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/docsummary/summary_field_converter - src/tests/extractkeywords src/tests/juniper ) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 7265dd89be4..a00592400b5 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -2,9 +2,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> @@ -40,7 +38,7 @@ struct AttributeCombinerTest : public ::testing::Test std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; AttributeCombinerTest(); - ~AttributeCombinerTest(); + ~AttributeCombinerTest() override; void set_field(const vespalib::string &field_name, bool filter_elements); void assertWritten(const vespalib::string &exp, uint32_t docId); }; diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp index 9143f17cb67..06a20563161 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -44,7 +44,7 @@ make_doc_type_repo() DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, doc_type_name, Struct(header_name), Struct(body_name)); - return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); + return std::make_unique<const DocumentTypeRepo>(builder.config()); } class DocumentIdDFWTest : public ::testing::Test diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 160e4cec973..22b3ae69165 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -3,18 +3,19 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/datatype/arraydatatype.h> #include <vespa/document/datatype/mapdatatype.h> +#include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/mapfieldvalue.h> +#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/document/fieldvalue//document.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> @@ -68,6 +69,7 @@ private: StructDataType::UP _elem_type; ArrayDataType _array_type; MapDataType _map_type; + WeightedSetDataType _wset_type; StructFieldValue::UP make_elem_value(const std::string& name, int weight) const { auto result = std::make_unique<StructFieldValue>(*_elem_type); @@ -82,11 +84,13 @@ public: _doc_type("test"), _elem_type(make_struct_elem_type()), _array_type(*_elem_type), - _map_type(*DataType::STRING, *_elem_type) + _map_type(*DataType::STRING, *_elem_type), + _wset_type(*DataType::STRING, false, false) { _doc_type.addField(Field("array", _array_type)); _doc_type.addField(Field("map", _map_type)); _doc_type.addField(Field("map2", _map_type)); + _doc_type.addField(Field("wset", _wset_type)); auto* result_class = _config.AddResultClass("test", class_id); EXPECT_TRUE(result_class->AddConfigEntry("array", ResType::RES_JSONSTRING)); @@ -118,6 +122,13 @@ public: map2_value.put(StringFieldValue("dummy"), *make_elem_value("dummy", 2)); doc->setValue("map2", map2_value); } + { + WeightedSetFieldValue wset_value(_wset_type); + wset_value.add(StringFieldValue("a"), 13); + wset_value.add(StringFieldValue("b"), 15); + wset_value.add(StringFieldValue("c"), 17); + doc->setValue("wset", wset_value); + } return std::make_unique<DocsumStoreDocument>(std::move(doc)); } }; @@ -161,8 +172,8 @@ public: { } ~StateCallback() override; - void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} - void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} + void FillSummaryFeatures(GetDocsumsState&) override {} + void FillRankFeatures(GetDocsumsState&) override {} std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields&) override { auto result = std::make_unique<MatchingElements>(); result->add_matching_elements(doc_id, _field_name, _matching_elements); @@ -243,6 +254,16 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value) expect_filtered("map", {0, 1, 100}, "[]"); } +TEST_F(MatchedElementsFilterTest, filter_elements_in_weighed_set_field_value) +{ + expect_filtered("wset", {}, "[]"); + expect_filtered("wset", {0}, "[{'item':'a','weight':13}]"); + expect_filtered("wset", {1}, "[{'item':'b','weight':15}]"); + expect_filtered("wset", {2}, "[{'item':'c','weight':17}]"); + expect_filtered("wset", {0, 1, 2}, "[{'item':'a','weight':13},{'item':'b','weight':15},{'item':'c','weight':17}]"); + expect_filtered("wset", {0, 1, 100}, "[]"); +} + TEST_F(MatchedElementsFilterTest, matching_elements_fields_is_setup_for_map_field_value) { { diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 60584b26e31..f2e949cbddf 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -9,7 +9,6 @@ #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/test/slime_value.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/juniper/rpinterface.h> @@ -52,9 +51,9 @@ struct MyEnvironment : IDocsumEnvironment { MyEnvironment() : attribute_man(0) {} - IAttributeManager *getAttributeManager() override { return attribute_man; } + const IAttributeManager *getAttributeManager() const override { return attribute_man; } string lookupIndex(const string &s) const override { return s; } - juniper::Juniper *getJuniper() override { return 0; } + const juniper::Juniper *getJuniper() const override { return nullptr; } }; class MyAttributeContext : public IAttributeContext { @@ -98,7 +97,7 @@ public: } IAttributeContext::UP createContext() const override { - return IAttributeContext::UP(new MyAttributeContext(_attr)); + return std::make_unique<MyAttributeContext>(_attr); } std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override { @@ -107,8 +106,8 @@ public: }; struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { - virtual void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} - virtual void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} + virtual void FillSummaryFeatures(GetDocsumsState&) override {} + virtual void FillRankFeatures(GetDocsumsState&) override {} std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &) override { abort(); } }; diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 445b08570a6..d12223d5cf4 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -14,13 +14,15 @@ #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> +#include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/util/size_literals.h> using namespace vespalib::slime::convenience; using namespace search::docsummary; +using vespalib::slime::BinaryFormat; using search::MatchingElements; using document::ByteFieldValue; using document::DataType; @@ -40,43 +42,32 @@ using document::StructFieldValue; namespace { -struct FieldBlock { - Slime slime; - search::RawBuf binary; - - explicit FieldBlock(const vespalib::string &jsonInput) - : slime(), binary(1024) - { - size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime); - EXPECT_TRUE(used > 0); - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::BinaryFormat::encode(slime, adapter); - } - const char *data() const { return binary.GetDrainPos(); } - size_t dataLen() const { return binary.GetUsedLen(); } -}; - struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { std::unique_ptr<DynamicDocsumWriter> writer; StructDataType int_pair_type; DocumentType doc_type; GetDocsumsState state; + bool fail_get_mapped_docsum; + bool empty_get_mapped_docsum; DocsumFixture(); - ~DocsumFixture(); + ~DocsumFixture() override; void getDocsum(Slime &slime) { - uint32_t classId; - search::RawBuf buf(4_Ki); - writer->WriteDocsum(1u, &state, this, &buf); - ASSERT_GREATER(buf.GetUsedLen(), sizeof(classId)); - memcpy(&classId, buf.GetDrainPos(), sizeof(classId)); - buf.Drain(sizeof(classId)); - EXPECT_EQUAL(classId, SLIME_MAGIC_ID); - EXPECT_GREATER(vespalib::slime::BinaryFormat - ::decode(Memory(buf.GetDrainPos(), buf.GetUsedLen()), slime), 0u); + Slime slimeOut; + SlimeInserter inserter(slimeOut); + writer->WriteDocsum(1u, &state, this, inserter); + vespalib::SmartBuffer buf(4_Ki); + BinaryFormat::encode(slimeOut, buf); + EXPECT_GREATER(BinaryFormat::decode(buf.obtain(), slime), 0u); } uint32_t getNumDocs() const override { return 2; } std::unique_ptr<const IDocsumStoreDocument> getMappedDocsum(uint32_t docid) override { EXPECT_EQUAL(1u, docid); + if (fail_get_mapped_docsum) { + return {}; + } + if (empty_get_mapped_docsum) { + return std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>()); + } auto doc = std::make_unique<Document>(doc_type, DocumentId("id:test:test::0")); doc->setValue("int_field", IntFieldValue(4)); doc->setValue("short_field", ShortFieldValue(2)); @@ -96,8 +87,8 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { } return std::make_unique<DocsumStoreDocument>(std::move(doc)); } - void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } - void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } + void FillSummaryFeatures(GetDocsumsState&) override { } + void FillRankFeatures(GetDocsumsState&) override { } std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields &) override { abort(); } }; @@ -106,11 +97,13 @@ DocsumFixture::DocsumFixture() : writer(), int_pair_type("int_pair"), doc_type("test"), - state(*this) + state(*this), + fail_get_mapped_docsum(false), + empty_get_mapped_docsum(false) { auto config = std::make_unique<ResultConfig>(); ResultClass *cfg = config->AddResultClass("default", 0); - EXPECT_TRUE(cfg != 0); + EXPECT_TRUE(cfg != nullptr); EXPECT_TRUE(cfg->AddConfigEntry("int_field", RES_INT)); EXPECT_TRUE(cfg->AddConfigEntry("short_field", RES_SHORT)); EXPECT_TRUE(cfg->AddConfigEntry("byte_field", RES_BYTE)); @@ -139,7 +132,7 @@ DocsumFixture::DocsumFixture() doc_type.addField(Field("longdata_field", *DataType::RAW)); doc_type.addField(Field("int_pair_field", int_pair_type)); } -DocsumFixture::~DocsumFixture() {} +DocsumFixture::~DocsumFixture() = default; } // namespace <unnamed> @@ -159,4 +152,29 @@ TEST_FF("require that docsum can be written as slime", DocsumFixture(), Slime()) EXPECT_EQUAL(f2.get()["int_pair_field"]["bar"].asLong(), 2u); } +TEST_FF("require that unknown summary class gives empty slime", DocsumFixture(), Slime()) +{ + f1.state._args.setResultClassName("unknown"); + f1.getDocsum(f2); + EXPECT_TRUE(f2.get().valid()); + EXPECT_EQUAL(vespalib::slime::NIX::ID, f2.get().type().getId()); +} + +TEST_FF("require that failure to retrieve docsum store document gives empty slime", DocsumFixture(), Slime()) +{ + f1.fail_get_mapped_docsum = true; + f1.getDocsum(f2); + EXPECT_TRUE(f2.get().valid()); + EXPECT_EQUAL(vespalib::slime::NIX::ID, f2.get().type().getId()); +} + +TEST_FF("require that empty docsum store document gives empty object", DocsumFixture(), Slime()) +{ + f1.empty_get_mapped_docsum = true; + f1.getDocsum(f2); + EXPECT_TRUE(f2.get().valid()); + EXPECT_EQUAL(vespalib::slime::OBJECT::ID, f2.get().type().getId()); + EXPECT_EQUAL(0u, f2.get().fields()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp b/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp index 06766ba370a..1d7795d26dc 100644 --- a/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp +++ b/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp @@ -132,7 +132,7 @@ FieldBlock::FieldBlock(const vespalib::string &jsonInput) vespalib::slime::BinaryFormat::encode(slime, adapter); } -FieldBlock::~FieldBlock() {} +FieldBlock::~FieldBlock() = default; class Test : public vespalib::TestApp { std::unique_ptr<Schema> _schema; @@ -242,7 +242,7 @@ DocumenttypesConfig getDocumenttypesConfig() { } Test::Test() : - _documentRepo(new DocumentTypeRepo(getDocumenttypesConfig())), + _documentRepo(std::make_unique<DocumentTypeRepo>(getDocumenttypesConfig())), _documentType(_documentRepo->getDocumentType("indexingdocument")), _fixedRepo(*_documentRepo, *_documentType) { @@ -285,8 +285,8 @@ Test::Main() } void Test::setUp() { - _schema.reset(new Schema); - _summarymap.reset(new SummarymapConfigBuilder); + _schema = std::make_unique<Schema>(); + _summarymap = std::make_unique<SummarymapConfigBuilder>(); } void Test::tearDown() { @@ -298,34 +298,26 @@ const DataType &Test::getDataType(const string &name) const { return *type; } -template <typename T> -std::unique_ptr<T> makeUP(T *p) { return std::unique_ptr<T>(p); } - StringFieldValue Test::makeAnnotatedString() { - SpanList *span_list = new SpanList; - SpanTree::UP tree(new SpanTree(SPANTREE_NAME, makeUP(span_list))); + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); // Annotations don't have to be added sequentially. - tree->annotate(span_list->add(makeUP(new Span(8, 3))), - makeUP(new Annotation(*TERM, - makeUP(new StringFieldValue( - "Annotation"))))); - tree->annotate(span_list->add(makeUP(new Span(0, 3))), *TERM); - tree->annotate(span_list->add(makeUP(new Span(4, 3))), *TERM); - tree->annotate(span_list->add(makeUP(new Span(4, 3))), - makeUP(new Annotation(*TERM, - makeUP(new StringFieldValue( - "Multiple"))))); - tree->annotate(span_list->add(makeUP(new Span(1, 2))), - makeUP(new Annotation(*TERM, - makeUP(new StringFieldValue( - "Overlap"))))); + tree->annotate(span_list->add(std::make_unique<Span>(8, 3)), + Annotation(*TERM, std::make_unique<StringFieldValue>("Annotation"))); + tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *TERM); + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), *TERM); + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), + Annotation(*TERM, std::make_unique<StringFieldValue>("Multiple"))); + tree->annotate(span_list->add(std::make_unique<Span>(1, 2)), + Annotation(*TERM, std::make_unique<StringFieldValue>("Overlap"))); StringFieldValue value("Foo Bar Baz"); setSpanTree(value, std::move(tree)); return value; } StringFieldValue Test::annotateTerm(const string &term) { - SpanTree::UP tree(new SpanTree(SPANTREE_NAME, makeUP(new Span(0, term.size())))); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::make_unique<Span>(0, term.size())); tree->annotate(tree->getRoot(), *TERM); StringFieldValue value(term); setSpanTree(value, std::move(tree)); @@ -339,11 +331,12 @@ void Test::setSpanTree(StringFieldValue & value, SpanTree::UP tree) { } StringFieldValue Test::makeAnnotatedChineseString() { - SpanList *span_list = new SpanList; - SpanTree::UP tree(new SpanTree(SPANTREE_NAME, makeUP(span_list))); + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); // These chinese characters each use 3 bytes in their UTF8 encoding. - tree->annotate(span_list->add(makeUP(new Span(0, 15))), *TERM); - tree->annotate(span_list->add(makeUP(new Span(15, 9))), *TERM); + tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *TERM); + tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *TERM); StringFieldValue value("我就是那个大灰狼"); setSpanTree(value, std::move(tree)); return value; @@ -660,7 +653,7 @@ void Test::requireThatLinguisticsAnnotationUsesDefaultDataTypes() { void Test::requireThatPredicateIsPrinted() { - std::unique_ptr<Slime> input(new Slime()); + auto input = std::make_unique<Slime>(); Cursor &obj = input->setObject(); obj.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET); obj.setString(Predicate::KEY, "foo"); diff --git a/searchsummary/src/tests/extractkeywords/.gitignore b/searchsummary/src/tests/extractkeywords/.gitignore deleted file mode 100644 index 1b50b24b284..00000000000 --- a/searchsummary/src/tests/extractkeywords/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*.core -.depend -Makefile -core -core.* -extractkeywordstest -searchsummary_extractkeywordstest_app diff --git a/searchsummary/src/tests/extractkeywords/CMakeLists.txt b/searchsummary/src/tests/extractkeywords/CMakeLists.txt deleted file mode 100644 index 802bff92544..00000000000 --- a/searchsummary/src/tests/extractkeywords/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchsummary_extractkeywordstest_app TEST - SOURCES - extractkeywordstest.cpp - simplequerystack.cpp - simplequerystackitem.cpp - DEPENDS - searchsummary -) -vespa_add_test(NAME searchsummary_extractkeywordstest_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/runtests.sh - DEPENDS searchsummary_extractkeywordstest_app) diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp deleted file mode 100644 index 724cf338497..00000000000 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "extractkeywordstest.h" -#include <vespa/vespalib/util/signalhandler.h> -#include <vespa/searchsummary/docsummary/keywordextractor.h> -#include "simplequerystack.h" -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/time.h> - -#define NUMTESTS 5 - -int -ExtractKeywordsTest::main(int argc, char **argv) -{ - int doTest[NUMTESTS]; - int low, high, accnum, num; - int indicator; - bool verify = false; - int multiplier = 1; - bool failed = false; - - if (argc == 1) - return Usage(argv[0]); - - // default initialize to not run any tests. - for (int n = 0; n < NUMTESTS; n++) - doTest[n] = 0; - - // parse the command line arguments - for (int i = 1; i < argc; i++) { - low = 0; - high = NUMTESTS - 1; - char *p = argv[i]; - - // Check if a multiplier is specified - if (*p == '*') { - p++; - accnum = 0; - while (*p != '\0') { - num = *p - '0'; - accnum = accnum * 10 + num; - p++; - } - multiplier = accnum; - continue; - } - - // Default is to run the tests specified, unless the first char is '/' - indicator = 1; - if (*p == '/') { - p++; - indicator = 0; - } - - // Find the first number - accnum = 0; - while (*p != '-' && *p != '\0') { - num = *p - '0'; - accnum = accnum * 10 + num; - p++; - } - if (accnum >= NUMTESTS) - continue; - low = accnum; - // Check for range operator - if (*p == '-') { - p++; - // Find the second number - accnum = 0; - while (*p != '\0') { - num = *p - '0'; - accnum = accnum * 10 + num; - p++; - } - if (accnum > 0) - high = accnum < NUMTESTS ? accnum : NUMTESTS-1; - } else - high = low; - - // Indicate the runrequest for the desired range. - for (int j = low; j <= high; j++) - doTest[j] = indicator; - } - - // Remove unused tests. - // doTest[1] = 0; - - // Remember time - if (multiplier > 1) { - printf("Running all tests %d times.\n", multiplier); - verify = false; - } else { - verify = true; - } - - int testCnt = 0; - - // init keyword extractor - _extractor = new search::docsummary::KeywordExtractor(nullptr); - _extractor->AddLegalIndexSpec("*"); - - vespalib::Timer timer; - - // Actually run the tests that we wanted. - for (int j = 0; j < multiplier; j++) - for (int k = 0; k < NUMTESTS; k++) - if (doTest[k] == 1) { - if (!RunTest(k, verify)) - failed = true; - testCnt++; - } - - // Print time taken - double timeTaken = vespalib::to_s(timer.elapsed())*1000.0; - - printf("Time taken : %f ms\n", timeTaken); - printf("Number of tests run: %d\n", testCnt); - double avgTestPrMSec = static_cast<double>(testCnt) / timeTaken; - printf("Tests pr Sec: %f\n", avgTestPrMSec * 1000.0); - - delete _extractor; - _extractor = nullptr; - - return failed ? 1 : 0; -} - -bool -ExtractKeywordsTest::ShowResult(int testNo, - const char *actual, const char *correct) -{ - const char *act_word = actual; - const char *cor_word = correct; - printf("%03d: ", testNo); - - while (*act_word != '\0') { - if (strcmp(act_word, cor_word) != 0) { - printf("fail. Keywords differ for act: %s, corr: %s\n", - act_word, cor_word); - return false; - } else { - act_word += strlen(act_word) + 1; - cor_word += strlen(cor_word) + 1; - } - } - if (*cor_word != '\0') { - printf("fail. actual list shorter than correct at %s\n", cor_word); - return false; - } - printf("ok\n"); - return true; -} - -/** - * - * @param testno The test to run. - * @param verify Verify the result of the test. - */ -bool -ExtractKeywordsTest::RunTest(int testno, bool verify) -{ - search::SimpleQueryStack stack; - search::RawBuf buf(32_Ki); - const char *correct = nullptr; - const char *keywords = nullptr; - - switch (testno) { - case 0: - { - // Simple term query - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foobar")); - - stack.AppendBuffer(&buf); - keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); - correct = "foobar\0\0"; - - if (verify) ShowResult(testno, keywords, correct); - free(const_cast<char *>(keywords)); - break; - } - - case 1: - { - // check that skipping these works also: - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_GEO_LOCATION_TERM, "no")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_NEAREST_NEIGHBOR, "no")); - // multi term query - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foobar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foo")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_OR, 3)); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_AND, 3)); - - stack.AppendBuffer(&buf); - keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); - correct = "bar\0foo\0foobar\0\0"; - - if (verify) ShowResult(testno, keywords, correct); - free(const_cast<char *>(keywords)); - break; - } - - case 2: - { - // phrase term query - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foobar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foo")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_PHRASE, 3, "index")); - - stack.AppendBuffer(&buf); - keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); - correct = "bar foo foobar\0\0"; - - if (verify) ShowResult(testno, keywords, correct); - free(const_cast<char *>(keywords)); - break; - } - - case 3: - { - // multiple phrase and term query - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "xyzzy")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "xyz")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_PHRASE, 2, "index")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foobar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foo")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_PHRASE, 3, "index")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "baz")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "zog")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_AND, 3)); - - stack.AppendBuffer(&buf); - keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); - correct = "zog\0baz\0bar foo foobar\0xyz xyzzy\0\0"; - - if (verify) ShowResult(testno, keywords, correct); - free(const_cast<char *>(keywords)); - break; - } - - case 4: - { - // phrase term query with wrong argument items - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foobar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foo")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_AND, 2)); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_PHRASE, 2, "index")); - - stack.AppendBuffer(&buf); - keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); - correct = "\0"; - - if (verify) ShowResult(testno, keywords, correct); - free(const_cast<char *>(keywords)); - break; - } - - default: - { - printf("%03d: no such test\n", testno); - return false; - } - } - - bool result = true; - /* - if (verify) { - result = ShowResult(testno, pq->GetStack(), correct); - delete correct; - } else { - result = true; - } - delete pq; - */ - return result; -} - -int -ExtractKeywordsTest::Usage(char *progname) -{ - printf("%s {testnospec}+\n\ - Where testnospec is:\n\ - num: single test\n\ - num-num: inclusive range (open range permitted)\n",progname); - printf("There are tests from %d to %d\n\n", 0, NUMTESTS-1); - return EXIT_FAILURE; -} - -int main(int argc, char** argv) { - vespalib::SignalHandler::PIPE.ignore(); - ExtractKeywordsTest tester; - return tester.main(argc, argv); -} diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.h b/searchsummary/src/tests/extractkeywords/extractkeywordstest.h deleted file mode 100644 index 6bd07d8a111..00000000000 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -namespace search::docsummary { class KeywordExtractor; } - -class ExtractKeywordsTest -{ -private: - ExtractKeywordsTest(const ExtractKeywordsTest &); - ExtractKeywordsTest& operator=(const ExtractKeywordsTest &); - - search::docsummary::KeywordExtractor *_extractor; - - int Usage(char *progname); - bool ShowResult(int testNo, const char *actual, const char *correct); - bool RunTest(int i, bool verify); - -public: - ExtractKeywordsTest() - : _extractor(nullptr) - {} - int main(int argc, char **argv); -}; - diff --git a/searchsummary/src/tests/extractkeywords/runtests.sh b/searchsummary/src/tests/extractkeywords/runtests.sh deleted file mode 100755 index 611b47dd888..00000000000 --- a/searchsummary/src/tests/extractkeywords/runtests.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -if $VALGRIND ./searchsummary_extractkeywordstest_app - -then - : -else - echo FAILED: searchsummary_extractkeywordstest_app test failed - exit 1 -fi - -if $VALGRIND ./searchsummary_extractkeywordstest_app - '*1000' -then - : -else - echo FAILED: searchsummary_extractkeywordstest_app test failed - exit 1 -fi - -echo SUCCESS: searchsummary_extractkeywordstest_app test completed diff --git a/searchsummary/src/tests/extractkeywords/simplequerystack.cpp b/searchsummary/src/tests/extractkeywords/simplequerystack.cpp deleted file mode 100644 index c96ef8a8455..00000000000 --- a/searchsummary/src/tests/extractkeywords/simplequerystack.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simplequerystack.h" -#include <vespa/vespalib/util/compress.h> - -#include <vespa/log/log.h> -LOG_SETUP(".search.simplequerystack"); - -namespace search { - -SimpleQueryStack::SimpleQueryStack() - : _stack(nullptr) -{ -} - -SimpleQueryStack::~SimpleQueryStack() -{ - delete _stack; -} - -void -SimpleQueryStack::Push(SimpleQueryStackItem *item) -{ - item->_next = _stack; - _stack = item; -} - -void -SimpleQueryStack::AppendBuffer(RawBuf *buf) const -{ - for (SimpleQueryStackItem *item = _stack; item != nullptr; item = item->_next) { - item->AppendBuffer(buf); - } -} - -} // namespace search diff --git a/searchsummary/src/tests/extractkeywords/simplequerystack.h b/searchsummary/src/tests/extractkeywords/simplequerystack.h deleted file mode 100644 index 0b61a41944b..00000000000 --- a/searchsummary/src/tests/extractkeywords/simplequerystack.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "simplequerystackitem.h" -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/stllike/string.h> - -namespace search { - -/** - * A stack of SimpleQueryStackItems. - * - * A simple stack consisting of a list of SimpleQueryStackItems. - * It is able to generate a binary encoding of itself - * to a RawBuf. - */ -class SimpleQueryStack -{ -private: - /** The top of the stack. */ - SimpleQueryStackItem *_stack; - -public: - SimpleQueryStack(const SimpleQueryStack &) = delete; - SimpleQueryStack& operator=(const SimpleQueryStack &) = delete; - /** - * Constructor for SimpleQueryStack. - */ - SimpleQueryStack(); - /** - * Destructor for SimpleQueryStack. - */ - ~SimpleQueryStack(); - /** - * Push an item on the stack. - * @param item The SimpleQueryStackItem to push. - */ - void Push(SimpleQueryStackItem *item); - - /** - * Encode the contents of the stack in a binary buffer. - * @param buf Pointer to a buffer containing the encoded contents. - */ - void AppendBuffer(RawBuf *buf) const; -}; - -} // namespace search - diff --git a/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp b/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp deleted file mode 100644 index 65815f86251..00000000000 --- a/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simplequerystackitem.h" -#include <vespa/vespalib/objects/nbo.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <cassert> - -namespace search { - -SimpleQueryStackItem::SimpleQueryStackItem() - : _next(NULL), - _arg1(0), - _arg2(0), - _arg3(0), - _type(ITEM_UNDEF), - _arity(0), - _indexName(), - _term() -{} - -namespace { - -void assert_term_type(ParseItem::ItemType type) { - assert(type == ParseItem::ITEM_TERM || - type == ParseItem::ITEM_NUMTERM || - type == ParseItem::ITEM_NEAREST_NEIGHBOR || - type == ParseItem::ITEM_GEO_LOCATION_TERM || - type == ParseItem::ITEM_PREFIXTERM || - type == ParseItem::ITEM_SUBSTRINGTERM || - type == ParseItem::ITEM_SUFFIXTERM || - type == ParseItem::ITEM_PURE_WEIGHTED_STRING || - type == ParseItem::ITEM_PURE_WEIGHTED_LONG || - type == ParseItem::ITEM_EXACTSTRINGTERM || - type == ParseItem::ITEM_PREDICATE_QUERY); - (void) type; -} - -void assert_arity_type(ParseItem::ItemType type) { - // types with arity, but without an index name: - assert(type == ParseItem::ITEM_OR || - type == ParseItem::ITEM_WEAK_AND || - type == ParseItem::ITEM_EQUIV || - type == ParseItem::ITEM_AND || - type == ParseItem::ITEM_NOT || - type == ParseItem::ITEM_RANK || - type == ParseItem::ITEM_ANY || - type == ParseItem::ITEM_NEAR || - type == ParseItem::ITEM_ONEAR); - (void) type; -} - -void assert_arity_and_index_type(ParseItem::ItemType type) { - // types with arity and an index name: - assert(type == ParseItem::ITEM_PHRASE || - type == ParseItem::ITEM_SAME_ELEMENT || - type == ParseItem::ITEM_WEIGHTED_SET || - type == ParseItem::ITEM_DOT_PRODUCT || - type == ParseItem::ITEM_WAND || - type == ParseItem::ITEM_WORD_ALTERNATIVES); - (void) type; -} - -int64_t term_as_n64(vespalib::stringref term) { - int64_t tmp; - vespalib::asciistream generatedTerm(term); - generatedTerm >> tmp; - return vespalib::nbo::n2h(tmp); -} - -} // namespace <unnamed> - - -SimpleQueryStackItem::SimpleQueryStackItem(ItemType type, int arity) : SimpleQueryStackItem() -{ - assert_arity_type(type); - SetType(type); - _arity = arity; -} - -SimpleQueryStackItem::SimpleQueryStackItem(ItemType type, int arity, const char *idx) : SimpleQueryStackItem() -{ - assert_arity_and_index_type(type); - SetType(type); - _arity = arity; - SetIndex(idx); -} - -SimpleQueryStackItem::SimpleQueryStackItem(ItemType type, const char *term) : SimpleQueryStackItem() -{ - assert_term_type(type); - SetType(type); - SetTerm(term); -} - -SimpleQueryStackItem::~SimpleQueryStackItem() -{ - delete _next; -} - -void -SimpleQueryStackItem::AppendBuffer(RawBuf *buf) const -{ - // Calculate lengths - uint32_t indexLen = _indexName.size(); - uint32_t termLen = _term.size(); - double nboVal = 0.0; - - // Put the values into the buffer. - buf->append(_type); - switch (Type()) { - case ITEM_OR: - case ITEM_EQUIV: - case ITEM_AND: - case ITEM_NOT: - case ITEM_RANK: - case ITEM_ANY: - buf->appendCompressedPositiveNumber(_arity); - break; - case ITEM_NEAR: - case ITEM_ONEAR: - buf->appendCompressedPositiveNumber(_arity); - buf->appendCompressedPositiveNumber(_arg1); - break; - case ITEM_SAME_ELEMENT: - case ITEM_WEIGHTED_SET: - case ITEM_DOT_PRODUCT: - case ITEM_PHRASE: - buf->appendCompressedPositiveNumber(_arity); - buf->appendCompressedPositiveNumber(indexLen); - buf->append(_indexName.c_str(), indexLen); - break; - case ITEM_WORD_ALTERNATIVES: - buf->appendCompressedPositiveNumber(indexLen); - buf->append(_indexName.c_str(), indexLen); - buf->appendCompressedPositiveNumber(_arity); - break; - case ITEM_WEAK_AND: - buf->appendCompressedPositiveNumber(_arity); - buf->appendCompressedPositiveNumber(_arg1); - buf->appendCompressedPositiveNumber(indexLen); - buf->append(_indexName.c_str(), indexLen); - break; - case ITEM_WAND: - buf->appendCompressedPositiveNumber(_arity); - buf->appendCompressedPositiveNumber(indexLen); - buf->append(_indexName.c_str(), indexLen); - buf->appendCompressedPositiveNumber(_arg1); // targetNumHits - nboVal = vespalib::nbo::n2h(_arg2); - buf->append(&nboVal, sizeof(nboVal)); // scoreThreshold - nboVal = vespalib::nbo::n2h(_arg3); - buf->append(&nboVal, sizeof(nboVal)); // thresholdBoostFactor - break; - case ITEM_TERM: - case ITEM_NUMTERM: - case ITEM_GEO_LOCATION_TERM: - case ITEM_PREFIXTERM: - case ITEM_SUBSTRINGTERM: - case ITEM_EXACTSTRINGTERM: - case ITEM_SUFFIXTERM: - case ITEM_REGEXP: - case ITEM_FUZZY: - buf->appendCompressedPositiveNumber(indexLen); - buf->append(_indexName.c_str(), indexLen); - buf->appendCompressedPositiveNumber(termLen); - buf->append(_term.c_str(), termLen); - break; - case ITEM_TRUE: - case ITEM_FALSE: - // no content - break; - case ITEM_PURE_WEIGHTED_STRING: - buf->appendCompressedPositiveNumber(termLen); - buf->append(_term.c_str(), termLen); - break; - case ITEM_PURE_WEIGHTED_LONG: - { - int64_t tmp = term_as_n64(_term); - buf->append(&tmp, sizeof(int64_t)); - } - break; - case ITEM_NEAREST_NEIGHBOR: - buf->appendCompressedPositiveNumber(indexLen); - buf->append(_indexName.c_str(), indexLen); - buf->appendCompressedPositiveNumber(termLen); - buf->append(_term.c_str(), termLen); - buf->appendCompressedPositiveNumber(_arg1); // targetNumHits - buf->appendCompressedPositiveNumber(_arg2); // allow_approximate - buf->appendCompressedPositiveNumber(_arg3); // explore_additional_hits - break; - case ITEM_MULTI_TERM: // TODO: handle - case ITEM_PREDICATE_QUERY: // not handled at all here - case ITEM_UNDEF: - abort(); - break; - } -} - -} diff --git a/searchsummary/src/tests/extractkeywords/simplequerystackitem.h b/searchsummary/src/tests/extractkeywords/simplequerystackitem.h deleted file mode 100644 index 58864e18444..00000000000 --- a/searchsummary/src/tests/extractkeywords/simplequerystackitem.h +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/searchlib/query/weight.h> -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/parsequery/parse.h> - -namespace search { - -/** - * An item on the simple query stack. - * - * An object of this class represents a single item - * on the simple query stack. It has a type, which corresponds - * to the different query stack execution operations. It also - * provides an arity, and the string values indexName and term, to - * accomodate the different needs of the operations. - */ -class SimpleQueryStackItem : public ParseItem -{ -private: - SimpleQueryStackItem(const SimpleQueryStackItem &) = delete; - SimpleQueryStackItem& operator=(const SimpleQueryStackItem &) = delete; - SimpleQueryStackItem(); -public: - /** Pointer to next item in a linked list. */ - SimpleQueryStackItem *_next; - -private: - uint32_t _arg1; - double _arg2; - double _arg3; - ItemType _type; - -public: - ItemType Type() const { return _type; } - - /** The number of operands for the operation. */ - uint32_t _arity; - /** The name of the specified index, or empty if no index. */ - vespalib::string _indexName; - /** The specified search term. */ - vespalib::string _term; - -/** - * Overloaded constructor for SimpleQueryStackItem. Used primarily for - * the operators, or phrase without indexName. - * - * @param type The type of the SimpleQueryStackItem. - * @param arity The arity of the operation indicated by the SimpleQueryStackItem. - */ - SimpleQueryStackItem(ItemType type, int arity); - -/** - * Overloaded constructor for SimpleQueryStackItem. Used for PHRASEs. - * - * @param type The type of the SimpleQueryStackItem. - * @param arity The arity of the operation indicated by the SimpleQueryStackItem. - * @param idx The name of the index of the SimpleQueryStackItem. - */ - SimpleQueryStackItem(ItemType type, int arity, const char *index); - -/** - * Overloaded constructor for SimpleQueryStackItem. Used for TERMs without index. - * - * @param type The type of the SimpleQueryStackItem. - * @param term The actual term string of the SimpleQueryStackItem. - */ - SimpleQueryStackItem(ItemType type, const char *term); - -/** - * Destructor for SimpleQueryStackItem. - */ - ~SimpleQueryStackItem(); - -/** - * Set the value of the _term field. - * @param term The string to set the _term field to. - */ - void SetTerm(const char *term) { _term = term; } - -/** - * Set the value of the _indexName field. - * @param idx The string to set the _indexName field to. - */ - void SetIndex(const char *index) { _indexName = index; } - - /** - * Set the type of the operator. Use this with caution, - * as this changes the semantics of the item. - * - * @param type The new type. - */ - void SetType(ItemType type) { - _type = type; - } - - /** - * Encode the item in a binary buffer. - * @param buf Pointer to a buffer containing the encoded contents. - */ - void AppendBuffer(RawBuf *buf) const; -}; - -} diff --git a/searchsummary/src/tests/juniper/testenv.cpp b/searchsummary/src/tests/juniper/testenv.cpp index 769c24b829c..cc6a6458376 100644 --- a/searchsummary/src/tests/juniper/testenv.cpp +++ b/searchsummary/src/tests/juniper/testenv.cpp @@ -106,9 +106,9 @@ PropertyMap::set(const char *name, const char *value) const char * -PropertyMap::GetProperty(const char* name, const char* def) +PropertyMap::GetProperty(const char* name, const char* def) const { - std::map<std::string, std::string>::iterator res = _map.find(std::string(name)); + auto res = _map.find(std::string(name)); if (res != _map.end()) { return res->second.c_str(); } diff --git a/searchsummary/src/tests/juniper/testenv.h b/searchsummary/src/tests/juniper/testenv.h index a43f4a11bec..f723fb96602 100644 --- a/searchsummary/src/tests/juniper/testenv.h +++ b/searchsummary/src/tests/juniper/testenv.h @@ -57,7 +57,7 @@ public: PropertyMap(); ~PropertyMap(); PropertyMap &set(const char *name, const char *value); - const char* GetProperty(const char* name, const char* def = NULL) override; + const char* GetProperty(const char* name, const char* def = nullptr) const override; }; diff --git a/searchsummary/src/vespa/juniper/IJuniperProperties.h b/searchsummary/src/vespa/juniper/IJuniperProperties.h index 63ada17684c..4902d3d561d 100644 --- a/searchsummary/src/vespa/juniper/IJuniperProperties.h +++ b/searchsummary/src/vespa/juniper/IJuniperProperties.h @@ -15,7 +15,7 @@ public: * @param def A default value for the property if not found in configuration * @return The value of the property or @param def if no such property is set */ - virtual const char* GetProperty(const char* name, const char* def = nullptr) = 0; + virtual const char* GetProperty(const char* name, const char* def = nullptr) const = 0; - virtual ~IJuniperProperties() {}; + virtual ~IJuniperProperties() = default; }; diff --git a/searchsummary/src/vespa/juniper/Matcher.cpp b/searchsummary/src/vespa/juniper/Matcher.cpp index 32f966f4571..76f4a17ad1c 100644 --- a/searchsummary/src/vespa/juniper/Matcher.cpp +++ b/searchsummary/src/vespa/juniper/Matcher.cpp @@ -478,8 +478,7 @@ std::string Matcher::GetLog() } -SummaryDesc* Matcher::CreateSummaryDesc(size_t length, size_t min_length, int max_matches, - int surround_len) +SummaryDesc* Matcher::CreateSummaryDesc(size_t length, size_t min_length, int max_matches, int surround_len) { // No point in processing this document if no keywords found at all: if (TotalHits() <= 0) return NULL; diff --git a/searchsummary/src/vespa/juniper/config.cpp b/searchsummary/src/vespa/juniper/config.cpp index a82a8d74b8a..3daebfd1ea8 100644 --- a/searchsummary/src/vespa/juniper/config.cpp +++ b/searchsummary/src/vespa/juniper/config.cpp @@ -10,7 +10,7 @@ namespace juniper { -Config::Config(const char* config_name, Juniper & juniper) : +Config::Config(const char* config_name, const Juniper & juniper) : _docsumparams(), _matcherparams(), _sumconf(nullptr), diff --git a/searchsummary/src/vespa/juniper/config.h b/searchsummary/src/vespa/juniper/config.h index a9dabdd91d1..51e2c67cfae 100644 --- a/searchsummary/src/vespa/juniper/config.h +++ b/searchsummary/src/vespa/juniper/config.h @@ -16,7 +16,7 @@ class Juniper; class Config { public: - Config(const char* config_name, Juniper & juniper); + Config(const char* config_name, const Juniper & juniper); ~Config(); const char* GetProp(const char* name, const char* def); @@ -26,7 +26,7 @@ public: private: std::string _config_name; - Juniper & _juniper; + const Juniper& _juniper; Config(Config &); Config &operator=(Config &); diff --git a/searchsummary/src/vespa/juniper/juniperparams.cpp b/searchsummary/src/vespa/juniper/juniperparams.cpp index 2ee0f3c31f6..e5a63440fd6 100644 --- a/searchsummary/src/vespa/juniper/juniperparams.cpp +++ b/searchsummary/src/vespa/juniper/juniperparams.cpp @@ -102,7 +102,7 @@ size_t MatcherParams::StemMinLength() const { return _stem_min; } size_t MatcherParams::StemMaxExtend() const { return _stem_extend; } -MatcherParams& MatcherParams::SetWordFolder(Fast_WordFolder* wordfolder) +MatcherParams& MatcherParams::SetWordFolder(const Fast_WordFolder* wordfolder) { _wordfolder = wordfolder; return *this; diff --git a/searchsummary/src/vespa/juniper/juniperparams.h b/searchsummary/src/vespa/juniper/juniperparams.h index 415c254b3f0..77422b02677 100644 --- a/searchsummary/src/vespa/juniper/juniperparams.h +++ b/searchsummary/src/vespa/juniper/juniperparams.h @@ -67,7 +67,7 @@ public: MatcherParams& SetStemMaxExtend(size_t stem_extend); size_t StemMaxExtend() const; - MatcherParams& SetWordFolder(Fast_WordFolder* wordfolder); + MatcherParams& SetWordFolder(const Fast_WordFolder* wordfolder); const Fast_WordFolder* WordFolder() const noexcept { return _wordfolder; } MatcherParams& SetProximityFactor(double factor); @@ -79,7 +79,7 @@ private: size_t _max_match_candidates; size_t _stem_min; size_t _stem_extend; - Fast_WordFolder* _wordfolder; // The wordfolder object needed as 1st parameter to folderfun + const Fast_WordFolder* _wordfolder; // The wordfolder object needed as 1st parameter to folderfun double _proximity_factor; }; diff --git a/searchsummary/src/vespa/juniper/propreader.cpp b/searchsummary/src/vespa/juniper/propreader.cpp index 99a6e580126..bd20c885f6c 100644 --- a/searchsummary/src/vespa/juniper/propreader.cpp +++ b/searchsummary/src/vespa/juniper/propreader.cpp @@ -80,7 +80,7 @@ void PropReader::Process(const char* filename) } -const char* PropReader::GetProperty(const char* name, const char* def) +const char* PropReader::GetProperty(const char* name, const char* def) const { const char* v = _keymap.Lookup(name, def); LOG(debug, "Parameter lookup :%s: value :%s:", name, v); diff --git a/searchsummary/src/vespa/juniper/propreader.h b/searchsummary/src/vespa/juniper/propreader.h index fbc6f53bfb1..45557716cd0 100644 --- a/searchsummary/src/vespa/juniper/propreader.h +++ b/searchsummary/src/vespa/juniper/propreader.h @@ -11,7 +11,7 @@ class PropReader : public IJuniperProperties { public: PropReader(const char* filename); - const char* GetProperty(const char* name, const char* def = NULL) override; + const char* GetProperty(const char* name, const char* def = nullptr) const override; void UpdateProperty(const char* name, const char* value); ~PropReader() {} protected: diff --git a/searchsummary/src/vespa/juniper/result.cpp b/searchsummary/src/vespa/juniper/result.cpp index fddc5d65c86..46e33209d52 100644 --- a/searchsummary/src/vespa/juniper/result.cpp +++ b/searchsummary/src/vespa/juniper/result.cpp @@ -8,6 +8,7 @@ #include "Matcher.h" #include "config.h" #include "appender.h" +#include <vespa/vespalib/util/size_literals.h> #include <vespa/log/log.h> LOG_SETUP(".juniper.result"); @@ -18,9 +19,9 @@ namespace juniper { class SummaryImpl : public Summary { public: - explicit SummaryImpl() : _text("") {} + explicit SummaryImpl() : _text() {} explicit SummaryImpl(const std::string& t) : _text(t) {} - ~SummaryImpl() {} + ~SummaryImpl() override = default; const char* Text() const override { return _text.c_str(); } size_t Length() const override { return _text.size(); } std::string _text; @@ -96,9 +97,7 @@ Result::Result(const Config& config, QueryHandle& qhandle, } } -Result::~Result() -{ -} +Result::~Result() = default; long Result::GetRelevancy() @@ -158,13 +157,12 @@ Summary* Result::GetTeaser(const Config* alt_config) ucs4_t *dst_end = dst + TOKEN_DSTLEN; const Fast_WordFolder *folder = _config->_matcherparams.WordFolder(); - text.reserve(_dynsum_len*2); + text.reserve(std::min(4_Ki, size_t(_dynsum_len*2))); if (src_end - src <= _dynsum_len) { a.append(text, src, src_end - src); src = src_end; // ensure while loop not run } - while (src < src_end) - { + while (src < src_end) { const char *startpos; size_t tokenLen; const char *old_src = src; diff --git a/searchsummary/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index c54ae654ec7..202b96a442d 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -73,12 +73,12 @@ Juniper::~Juniper() { } -std::unique_ptr<Config> Juniper::CreateConfig(const char* config_name) +std::unique_ptr<Config> Juniper::CreateConfig(const char* config_name) const { return std::unique_ptr<Config>(new Config(config_name, *this)); } -std::unique_ptr<QueryHandle> Juniper::CreateQueryHandle(const IQuery& fquery, const char* juniperoptions) +std::unique_ptr<QueryHandle> Juniper::CreateQueryHandle(const IQuery& fquery, const char* juniperoptions) const { return std::make_unique<QueryHandle>(fquery, juniperoptions, *_modifier); } diff --git a/searchsummary/src/vespa/juniper/rpinterface.h b/searchsummary/src/vespa/juniper/rpinterface.h index ee1f4e3a3d8..41a40e2c98d 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.h +++ b/searchsummary/src/vespa/juniper/rpinterface.h @@ -96,8 +96,8 @@ public: */ ~Juniper(); - Fast_WordFolder & getWordFolder() { return *_wordfolder; } - IJuniperProperties & getProp() { return *_props; } + const Fast_WordFolder & getWordFolder() const noexcept { return *_wordfolder; } + const IJuniperProperties & getProp() const noexcept { return *_props; } QueryModifier & getModifier() { return *_modifier; } /** Create a result processing configuration of Juniper for subsequent use @@ -111,7 +111,7 @@ public: * NULL if an error occurred. */ - std::unique_ptr<Config> CreateConfig(const char* config_name = "juniper"); + std::unique_ptr<Config> CreateConfig(const char* config_name = "juniper") const; /** Allocate a query handle for the given query for subsequent calls to Analyse * for different hits. Performs the necessary per query processing for Juniper. * @param query A query to start result processing for. @@ -122,7 +122,7 @@ public: * to the query language. * @return A unique pointer to a QueryHandle. */ - std::unique_ptr<QueryHandle> CreateQueryHandle(const IQuery& query, const char* juniperoptions); + std::unique_ptr<QueryHandle> CreateQueryHandle(const IQuery& query, const char* juniperoptions) const; /** Add an rewriter for all terms that are prefixed with the given index. * When Juniper encounter a term in the query tagged with this index, diff --git a/searchsummary/src/vespa/juniper/sumdesc.cpp b/searchsummary/src/vespa/juniper/sumdesc.cpp index fcee1eb605f..d6ac5e6e416 100644 --- a/searchsummary/src/vespa/juniper/sumdesc.cpp +++ b/searchsummary/src/vespa/juniper/sumdesc.cpp @@ -325,6 +325,7 @@ SummaryDesc::SummaryDesc(Matcher* matcher, ssize_t length, ssize_t min_length, locate_accidential_matches(); } +SummaryDesc::~SummaryDesc() = default; void SummaryDesc::locate_accidential_matches() diff --git a/searchsummary/src/vespa/juniper/sumdesc.h b/searchsummary/src/vespa/juniper/sumdesc.h index d91bf160e04..a0440a60fba 100644 --- a/searchsummary/src/vespa/juniper/sumdesc.h +++ b/searchsummary/src/vespa/juniper/sumdesc.h @@ -32,8 +32,8 @@ public: // Constructor that builds a description that can later be used to create // a suitable query in context / query highlight for the given matcher // in its current status: - SummaryDesc(Matcher* matcher, ssize_t length, ssize_t min_length, int max_matches, - int surround_len); + SummaryDesc(Matcher* matcher, ssize_t length, ssize_t min_length, int max_matches, int surround_len); + ~SummaryDesc(); /* Return a highlight tagged summary string * from this summary description diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index aec86d49d7d..963c94f7796 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -9,6 +9,7 @@ vespa_add_library(searchsummary_docsummary OBJECT copy_dfw.cpp docsumconfig.cpp docsum_field_writer.cpp + docsum_field_writer_factory.cpp docsum_store_document.cpp docsumstate.cpp docsumwriter.cpp @@ -24,6 +25,7 @@ vespa_add_library(searchsummary_docsummary OBJECT matched_elements_filter_dfw.cpp positionsdfw.cpp rankfeaturesdfw.cpp + res_config_entry.cpp res_type_utils.cpp resultclass.cpp resultconfig.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index 285148951fa..60e3bd6d815 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -364,7 +364,7 @@ create_multi_writer(const IAttributeVector& attr, bool filter_elements, std::sha } std::unique_ptr<DocsumFieldWriter> -AttributeDFWFactory::create(IAttributeManager& attr_mgr, +AttributeDFWFactory::create(const IAttributeManager& attr_mgr, const vespalib::string& attr_name, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index 88356ac783e..2b61803ebb3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -18,7 +18,7 @@ namespace search::docsummary { */ class AttributeDFWFactory { public: - static std::unique_ptr<DocsumFieldWriter> create(IAttributeManager& attr_mgr, + static std::unique_ptr<DocsumFieldWriter> create(const IAttributeManager& attr_mgr, const vespalib::string& attr_name, bool filter_elements = false, std::shared_ptr<MatchingElementsFields> matching_elems_fields = std::shared_ptr<MatchingElementsFields>()); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp new file mode 100644 index 00000000000..b3fa6c68b87 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -0,0 +1,121 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsum_field_writer_factory.h" +#include "attribute_combiner_dfw.h" +#include "copy_dfw.h" +#include "document_id_dfw.h" +#include "empty_dfw.h" +#include "geoposdfw.h" +#include "idocsumenvironment.h" +#include "juniperdfw.h" +#include "matched_elements_filter_dfw.h" +#include "positionsdfw.h" +#include "rankfeaturesdfw.h" +#include "summaryfeaturesdfw.h" +#include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/vespalib/util/exceptions.h> + +using vespalib::IllegalArgumentException; + +namespace search::docsummary { + +DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env) + : _use_v8_geo_positions(use_v8_geo_positions), + _env(env), + _matching_elems_fields(std::make_shared<MatchingElementsFields>()) +{ +} + +DocsumFieldWriterFactory::~DocsumFieldWriterFactory() = default; + +bool +DocsumFieldWriterFactory::has_attribute_manager() const noexcept +{ + return getEnvironment().getAttributeManager() != nullptr; +} + +std::unique_ptr<DocsumFieldWriter> +DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) +{ + rc = false; + std::unique_ptr<DocsumFieldWriter> fieldWriter; + if (overrideName == "dynamicteaser") { + if ( ! argument.empty() ) { + auto fw = std::make_unique<DynamicTeaserDFW>(getEnvironment().getJuniper()); + auto fw_ptr = fw.get(); + fieldWriter = std::move(fw); + rc = fw_ptr->Init(fieldName.c_str(), argument); + } else { + throw IllegalArgumentException("Missing argument"); + } + } else if (overrideName == "summaryfeatures") { + fieldWriter = std::make_unique<SummaryFeaturesDFW>(); + rc = true; + } else if (overrideName == "rankfeatures") { + fieldWriter = std::make_unique<RankFeaturesDFW>(); + rc = true; + } else if (overrideName == "empty") { + fieldWriter = std::make_unique<EmptyDFW>(); + rc = true; + } else if (overrideName == "copy") { + if ( ! argument.empty() ) { + fieldWriter = std::make_unique<CopyDFW>(argument); + rc = true; + } else { + throw IllegalArgumentException("Missing argument"); + } + } else if (overrideName == "absdist") { + if (has_attribute_manager()) { + fieldWriter = AbsDistanceDFW::create(argument.c_str(), getEnvironment().getAttributeManager()); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "positions") { + if (has_attribute_manager()) { + fieldWriter = PositionsDFW::create(argument.c_str(), getEnvironment().getAttributeManager(), _use_v8_geo_positions); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "geopos") { + if (has_attribute_manager()) { + fieldWriter = GeoPositionDFW::create(argument.c_str(), getEnvironment().getAttributeManager(), _use_v8_geo_positions); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "attribute") { + if (has_attribute_manager()) { + fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), argument); + rc = true; // Allow missing attribute vector + } + } else if (overrideName == "attributecombiner") { + if (has_attribute_manager()) { + auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); + const vespalib::string& source_field = argument.empty() ? fieldName : argument; + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, false, std::shared_ptr<MatchingElementsFields>()); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "matchedattributeelementsfilter") { + const vespalib::string& source_field = argument.empty() ? fieldName : argument; + if (has_attribute_manager()) { + auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); + if (attr_ctx->getAttribute(source_field) != nullptr) { + fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, _matching_elems_fields); + } else { + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, _matching_elems_fields); + } + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "matchedelementsfilter") { + const vespalib::string& source_field = argument.empty() ? fieldName : argument; + if (has_attribute_manager()) { + auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); + fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, _matching_elems_fields); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "documentid") { + fieldWriter = std::make_unique<DocumentIdDFW>(); + rc = true; + } else { + throw IllegalArgumentException("unknown override operation '" + overrideName + "' for field '" + fieldName + "'."); + } + return fieldWriter; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h new file mode 100644 index 00000000000..bab7153009d --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_docsum_field_writer_factory.h" + +namespace search { class MatchingElementsFields; } + +namespace search::docsummary { + +class IDocsumEnvironment; + +/* + * Factory class for creating docsum field writers. + */ +class DocsumFieldWriterFactory : public IDocsumFieldWriterFactory +{ + bool _use_v8_geo_positions; + const IDocsumEnvironment& _env; +protected: + std::shared_ptr<MatchingElementsFields> _matching_elems_fields; + const IDocsumEnvironment& getEnvironment() const noexcept { return _env; } + bool has_attribute_manager() const noexcept; +public: + DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env); + ~DocsumFieldWriterFactory() override; + std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 457d11e8f4b..0c2adcbfaa5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -1,20 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsumconfig.h" -#include "attribute_combiner_dfw.h" -#include "copy_dfw.h" +#include "docsum_field_writer_factory.h" #include "docsumwriter.h" -#include "document_id_dfw.h" -#include "empty_dfw.h" -#include "geoposdfw.h" -#include "idocsumenvironment.h" -#include "juniperdfw.h" -#include "matched_elements_filter_dfw.h" -#include "positionsdfw.h" -#include "rankfeaturesdfw.h" -#include "summaryfeaturesdfw.h" -#include <vespa/searchlib/common/matching_elements_fields.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> namespace search::docsummary { @@ -27,99 +15,20 @@ DynamicDocsumConfig::getResultConfig() const { return *_writer->GetResultConfig(); } -std::unique_ptr<DocsumFieldWriter> -DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields) +std::unique_ptr<IDocsumFieldWriterFactory> +DynamicDocsumConfig::make_docsum_field_writer_factory() { - const ResultConfig & resultConfig = getResultConfig(); - rc = false; - std::unique_ptr<DocsumFieldWriter> fieldWriter; - if (overrideName == "dynamicteaser") { - if ( ! argument.empty() ) { - auto fw = std::make_unique<DynamicTeaserDFW>(getEnvironment()->getJuniper()); - auto fw_ptr = fw.get(); - fieldWriter = std::move(fw); - rc = fw_ptr->Init(fieldName.c_str(), argument); - } else { - throw IllegalArgumentException("Missing argument"); - } - } else if (overrideName == "summaryfeatures") { - fieldWriter = std::make_unique<SummaryFeaturesDFW>(getEnvironment()); - rc = true; - } else if (overrideName == "rankfeatures") { - fieldWriter = std::make_unique<RankFeaturesDFW>(getEnvironment()); - rc = true; - } else if (overrideName == "empty") { - fieldWriter = std::make_unique<EmptyDFW>(); - rc = true; - } else if (overrideName == "copy") { - if ( ! argument.empty() ) { - fieldWriter = std::make_unique<CopyDFW>(argument); - rc = true; - } else { - throw IllegalArgumentException("Missing argument"); - } - } else if (overrideName == "absdist") { - if (getEnvironment()) { - fieldWriter = AbsDistanceDFW::create(argument.c_str(), getEnvironment()->getAttributeManager()); - rc = static_cast<bool>(fieldWriter); - } - } else if (overrideName == "positions") { - if (getEnvironment()) { - fieldWriter = PositionsDFW::create(argument.c_str(), getEnvironment()->getAttributeManager(), resultConfig.useV8geoPositions()); - rc = static_cast<bool>(fieldWriter); - } - } else if (overrideName == "geopos") { - if (getEnvironment()) { - fieldWriter = GeoPositionDFW::create(argument.c_str(), getEnvironment()->getAttributeManager(), resultConfig.useV8geoPositions()); - rc = static_cast<bool>(fieldWriter); - } - } else if (overrideName == "attribute") { - if (getEnvironment() && getEnvironment()->getAttributeManager()) { - fieldWriter = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), argument); - rc = true; // Allow missing attribute vector - } - } else if (overrideName == "attributecombiner") { - if (getEnvironment() && getEnvironment()->getAttributeManager()) { - auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); - const string& source_field = argument.empty() ? fieldName : argument; - fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, false, std::shared_ptr<MatchingElementsFields>()); - rc = static_cast<bool>(fieldWriter); - } - } else if (overrideName == "matchedattributeelementsfilter") { - const string& source_field = argument.empty() ? fieldName : argument; - if (getEnvironment() && getEnvironment()->getAttributeManager()) { - auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); - if (attr_ctx->getAttribute(source_field) != nullptr) { - fieldWriter = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), source_field, true, matching_elems_fields); - } else { - fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, matching_elems_fields); - } - rc = static_cast<bool>(fieldWriter); - } - } else if (overrideName == "matchedelementsfilter") { - const string& source_field = argument.empty() ? fieldName : argument; - if (getEnvironment() && getEnvironment()->getAttributeManager()) { - auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); - fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, matching_elems_fields); - rc = static_cast<bool>(fieldWriter); - } - } else if (overrideName == "documentid") { - fieldWriter = std::make_unique<DocumentIdDFW>(); - rc = true; - } else { - throw IllegalArgumentException("unknown override operation '" + overrideName + "' for field '" + fieldName + "'."); - } - return fieldWriter; + return std::make_unique<DocsumFieldWriterFactory>(getResultConfig().useV8geoPositions(), getEnvironment()); } void DynamicDocsumConfig::configure(const vespa::config::search::SummarymapConfig &cfg) { std::vector<string> strCfg; - auto matching_elems_fields = std::make_shared<MatchingElementsFields>(); + auto docsum_field_writer_factory = make_docsum_field_writer_factory(); for (const auto & o : cfg.override) { bool rc(false); - std::unique_ptr<DocsumFieldWriter> fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); + auto fieldWriter = docsum_field_writer_factory->create_docsum_field_writer(o.field, o.command, o.arguments, rc); if (rc && fieldWriter) { rc = _writer->Override(o.field.c_str(), std::move(fieldWriter)); // OBJECT HAND-OVER } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h index cb337f7f3d3..2b0f90a8e8b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h @@ -8,6 +8,7 @@ namespace search { class MatchingElementsFields; } namespace search::docsummary { class IDocsumEnvironment; +class IDocsumFieldWriterFactory; class DocsumFieldWriter; class DynamicDocsumWriter; class ResultConfig; @@ -15,7 +16,7 @@ class ResultConfig; class DynamicDocsumConfig { public: - DynamicDocsumConfig(IDocsumEnvironment * env, DynamicDocsumWriter * writer) : + DynamicDocsumConfig(const IDocsumEnvironment& env, DynamicDocsumWriter * writer) : _env(env), _writer(writer) { } @@ -23,14 +24,12 @@ public: void configure(const vespa::config::search::SummarymapConfig &cfg); protected: using string = vespalib::string; - IDocsumEnvironment * getEnvironment() { return _env; } + const IDocsumEnvironment& getEnvironment() { return _env; } const ResultConfig & getResultConfig() const; - virtual std::unique_ptr<DocsumFieldWriter> - createFieldWriter(const string & fieldName, const string & overrideName, - const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields); + virtual std::unique_ptr<IDocsumFieldWriterFactory> make_docsum_field_writer_factory(); private: - IDocsumEnvironment * _env; + const IDocsumEnvironment& _env; DynamicDocsumWriter * _writer; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index d3d09224f64..0ebcc20ef42 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -32,8 +32,8 @@ class DocsumFieldWriterState; class GetDocsumsStateCallback { public: - virtual void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; - virtual void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; + virtual void FillSummaryFeatures(GetDocsumsState& state) = 0; + virtual void FillRankFeatures(GetDocsumsState& state) = 0; virtual std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &matching_elems_fields) = 0; virtual ~GetDocsumsStateCallback() = default; GetDocsumsStateCallback(const GetDocsumsStateCallback &) = delete; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 27e8ecea4ca..39d4be1aa3b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -4,31 +4,21 @@ #include "docsumstate.h" #include "docsum_field_writer_state.h" #include "i_docsum_store_document.h" +#include "keywordextractor.h" #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/attribute/iattributemanager.h> -#include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.docsumwriter"); -using namespace vespalib::slime::convenience; using vespalib::Issue; +using vespalib::slime::ObjectInserter; +using vespalib::Memory; namespace search::docsummary { -uint32_t -IDocsumWriter::slime2RawBuf(const Slime & slime, RawBuf & buf) -{ - const uint32_t preUsed = buf.GetUsedLen(); - const uint32_t magic = SLIME_MAGIC_ID; - buf.append(&magic, sizeof(magic)); - SlimeOutputRawBufAdapter adapter(buf); - vespalib::slime::BinaryFormat::encode(slime, adapter); - return (buf.GetUsedLen() - preUsed); -} - DynamicDocsumWriter::ResolveClassInfo DynamicDocsumWriter::resolveClassInfo(vespalib::stringref outputClassName) const { @@ -46,23 +36,22 @@ DynamicDocsumWriter::resolveOutputClass(vespalib::stringref summaryClass) const if (oC == nullptr) { Issue::report("Illegal docsum class requested: %s, using empty docsum for documents", vespalib::string(summaryClass).c_str()); - result.mustSkip = true; } else { - result.outputClass = oC; const ResultClass::DynamicInfo *rcInfo = oC->getDynamicInfo(); if (rcInfo->_generateCnt == oC->GetNumEntries()) { LOG_ASSERT(rcInfo->_overrideCnt == rcInfo->_generateCnt); result.allGenerated = true; } } + result.outputClass = oC; return result; } void DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::slime::Inserter& topInserter) + IDocsumStore *docinfos, Inserter& topInserter) { - if (rci.mustSkip || rci.outputClass == nullptr) { + if (rci.outputClass == nullptr) { // Use empty docsum when illegal docsum class has been requested return; } @@ -81,6 +70,9 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, } else { // look up docsum entry auto doc = docinfos->getMappedDocsum(docid); + if (!doc) { + return; // Use empty docsum when document is gone + } // insert docsum blob vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { @@ -153,7 +145,7 @@ DynamicDocsumWriter::Override(const char *fieldName, std::unique_ptr<DocsumField void -DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *state) +DynamicDocsumWriter::InitState(const IAttributeManager & attrMan, GetDocsumsState *state) { state->_kwExtractor = _keywordExtractor.get(); state->_attrCtx = attrMan.createContext(); @@ -171,14 +163,11 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta } -uint32_t -DynamicDocsumWriter::WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) +void +DynamicDocsumWriter::WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, Inserter& inserter) { - vespalib::Slime slime; - vespalib::slime::SlimeInserter inserter(slime); ResolveClassInfo rci = resolveClassInfo(state->_args.getResultClassName()); insertDocsum(rci, docid, state, docinfos, inserter); - return slime2RawBuf(slime, *target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index bea6a747f84..909be169006 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -6,42 +6,42 @@ #include "resultclass.h" #include "resultconfig.h" #include "docsumstore.h" -#include "keywordextractor.h" #include "docsum_field_writer.h" -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/fastlib/text/unicodeutil.h> #include <vespa/fastlib/text/wordfolder.h> -namespace search { class IAttributeManager; } +namespace search { + class IAttributeManager; +} namespace vespalib { class Slime; } namespace search::docsummary { +class KeywordExtractor; + static constexpr uint32_t SLIME_MAGIC_ID = 0x55555555; class IDocsumWriter { public: + using Inserter = vespalib::slime::Inserter; struct ResolveClassInfo { - bool mustSkip; bool allGenerated; const ResultClass *outputClass; ResolveClassInfo() - : mustSkip(false), allGenerated(false), + : allGenerated(false), outputClass(nullptr) { } }; virtual ~IDocsumWriter() = default; - virtual void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) = 0; - virtual uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, search::RawBuf *target) = 0; + virtual void InitState(const search::IAttributeManager & attrMan, GetDocsumsState *state) = 0; + virtual void WriteDocsum(uint32_t docid, GetDocsumsState *state, + IDocsumStore *docinfos, Inserter & target) = 0; virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::slime::Inserter & target) = 0; + IDocsumStore *docinfos, Inserter & target) = 0; virtual ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName) const = 0; - - static uint32_t slime2RawBuf(const vespalib::Slime & slime, RawBuf & buf); }; //-------------------------------------------------------------------------- @@ -66,12 +66,12 @@ public: const ResultConfig *GetResultConfig() { return _resultConfig.get(); } bool Override(const char *fieldName, std::unique_ptr<DocsumFieldWriter> writer); - void InitState(search::IAttributeManager & attrMan, GetDocsumsState *state) override; - uint32_t WriteDocsum(uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, search::RawBuf *target) override; + void InitState(const search::IAttributeManager & attrMan, GetDocsumsState *state) override; + void WriteDocsum(uint32_t docid, GetDocsumsState *state, + IDocsumStore *docinfos, Inserter & inserter) override; void insertDocsum(const ResolveClassInfo & outputClassInfo, uint32_t docid, GetDocsumsState *state, - IDocsumStore *docinfos, vespalib::slime::Inserter & target) override; + IDocsumStore *docinfos, Inserter & inserter) override; ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName) const override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index f7762408904..957f49c0f86 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -4,7 +4,7 @@ #include "docsumwriter.h" #include "docsumstate.h" #include "i_docsum_store_document.h" -#include <vespa/document/fieldvalue/fieldvalue.h> +#include "keywordextractor.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/queryeval/split_float.h> #include <vespa/vespalib/objects/hexdump.h> @@ -286,7 +286,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const return rc; } -JuniperDFW::JuniperDFW(juniper::Juniper * juniper) +JuniperDFW::JuniperDFW(const juniper::Juniper * juniper) : _input_field_name(), _juniperConfig(), _juniper(juniper) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 975fd7f0a6a..474f329799b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -101,7 +101,7 @@ GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, GeoPositionDFW::UP GeoPositionDFW::create(const char *attribute_name, - IAttributeManager *attribute_manager, + const IAttributeManager *attribute_manager, bool useV8geoPositions) { GeoPositionDFW::UP ret; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h index db43592c0b4..1bc8b523160 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h @@ -17,7 +17,7 @@ public: typedef std::unique_ptr<GeoPositionDFW> UP; GeoPositionDFW(const vespalib::string & attrName, bool useV8geoPositions); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; - static UP create(const char *attribute_name, IAttributeManager *attribute_manager, bool useV8geoPositions); + static UP create(const char *attribute_name, const IAttributeManager *attribute_manager, bool useV8geoPositions); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h new file mode 100644 index 00000000000..927fef26d1a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vespa/vespalib/stllike/string.h> + +namespace search::docsummary { + +class DocsumFieldWriter; + +/* + * Factory interface class for creating docsum field writers. + */ +class IDocsumFieldWriterFactory +{ +public: + virtual ~IDocsumFieldWriterFactory() = default; + virtual std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) = 0; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/idocsumenvironment.h b/searchsummary/src/vespa/searchsummary/docsummary/idocsumenvironment.h index bc6a347590a..8af35d9ce3a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/idocsumenvironment.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/idocsumenvironment.h @@ -12,10 +12,10 @@ namespace search::docsummary { **/ class IDocsumEnvironment { public: - virtual search::IAttributeManager * getAttributeManager() = 0; + virtual const search::IAttributeManager * getAttributeManager() const = 0; virtual vespalib::string lookupIndex(const vespalib::string & s) const = 0; - virtual juniper::Juniper * getJuniper() = 0; - virtual ~IDocsumEnvironment() {} + virtual const juniper::Juniper * getJuniper() const = 0; + virtual ~IDocsumEnvironment() = default; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index 6a117fbc0cd..63642ed7543 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -20,12 +20,12 @@ public: const char *fieldName, const vespalib::string& inputField); protected: - explicit JuniperDFW(juniper::Juniper * juniper); + explicit JuniperDFW(const juniper::Juniper * juniper); ~JuniperDFW() override; vespalib::string _input_field_name; std::unique_ptr<juniper::Config> _juniperConfig; - juniper::Juniper *_juniper; + const juniper::Juniper *_juniper; private: bool IsGenerated() const override { return false; } JuniperDFW(const JuniperDFW &); @@ -39,7 +39,7 @@ public: bool Init(const char *fieldName, const vespalib::string& inputField) override; protected: - explicit JuniperTeaserDFW(juniper::Juniper * juniper) : JuniperDFW(juniper) { } + explicit JuniperTeaserDFW(const juniper::Juniper * juniper) : JuniperDFW(juniper) { } }; @@ -49,7 +49,7 @@ class DynamicTeaserDFW : public JuniperTeaserDFW vespalib::stringref input, GetDocsumsState *state) const; public: - explicit DynamicTeaserDFW(juniper::Juniper * juniper) : JuniperTeaserDFW(juniper) { } + explicit DynamicTeaserDFW(const juniper::Juniper * juniper) : JuniperTeaserDFW(juniper) { } void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp index b795679c7e6..8f6d6ed02ea 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp @@ -84,7 +84,7 @@ JuniperProperties::configure(const JuniperrcConfig &cfg) } const char * -JuniperProperties::GetProperty(const char *name, const char *def) +JuniperProperties::GetProperty(const char *name, const char *def) const { auto it = _properties.find(name); return it != _properties.end() ? it->second.c_str() : def; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h index 55f2e572b9b..88e819056cf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h @@ -26,7 +26,7 @@ public: /** * Constructs a juniper property object with default values set. */ - JuniperProperties(const vespa::config::search::summary::JuniperrcConfig &cfg); + explicit JuniperProperties(const vespa::config::search::summary::JuniperrcConfig &cfg); ~JuniperProperties() override; @@ -38,7 +38,7 @@ public: void configure(const vespa::config::search::summary::JuniperrcConfig &cfg); // Inherit doc from IJuniperProperties. - const char *GetProperty(const char *name, const char *def) override; + const char *GetProperty(const char *name, const char *def) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index e97017e79c4..e5e0d20832b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -17,7 +17,7 @@ bool useful(search::ParseItem::ItemCreator creator) } -KeywordExtractor::KeywordExtractor(IDocsumEnvironment * env) +KeywordExtractor::KeywordExtractor(const IDocsumEnvironment * env) : _env(env), _legalPrefixes(), _legalIndexes() @@ -33,7 +33,7 @@ KeywordExtractor::IsLegalIndexName(const char *idxName) const return _legalIndexes.find(idxName) != _legalIndexes.end(); } -KeywordExtractor::IndexPrefix::IndexPrefix(const char *prefix) +KeywordExtractor::IndexPrefix::IndexPrefix(const char *prefix) noexcept : _prefix(prefix) { } @@ -131,99 +131,4 @@ KeywordExtractor::IsLegalIndex(vespalib::stringref idxS) const IsLegalIndexName(resolvedIdxName.c_str())); } - -char * -KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const -{ - search::SimpleQueryStackDumpIterator si(buf); - char keywordstore[4_Ki]; // Initial storage for keywords buffer - search::RawBuf keywords(keywordstore, sizeof(keywordstore)); - - while (si.next()) { - search::ParseItem::ItemCreator creator = si.getCreator(); - switch (si.getType()) { - case search::ParseItem::ITEM_NOT: - /** - * @todo Must consider only the first argument on the stack. - * Difficult without recursion. - */ - break; - - case search::ParseItem::ITEM_PHRASE: - { - // Must take the next arity TERMS and put together - bool phraseterms_was_added = false; - int phraseterms = si.getArity(); - for (int i = 0; i < phraseterms; i++) { - si.next(); - search::ParseItem::ItemType newtype = si.getType(); - if (newtype != search::ParseItem::ITEM_TERM && - newtype != search::ParseItem::ITEM_NUMTERM) - { - // stack syntax error - // LOG(debug, "Extracting keywords found a non-term in a phrase"); - // making a clean escape. - keywords.reset(); - goto iteratorloopend; - } else { - if (!IsLegalIndex(si.getIndexName())) - continue; - // Found a term - vespalib::stringref term = si.getTerm(); - search::ParseItem::ItemCreator term_creator = si.getCreator(); - if ( !term.empty() && useful(term_creator)) { - // Actual term to add - if (phraseterms_was_added) { - // Not the first term in the phrase - keywords += " "; - } else { - phraseterms_was_added = true; - } - - keywords.append(term.data(), term.size()); - } - } - } - if (phraseterms_was_added) { - // Terms was added, so 0-terminate the string - keywords.append("\0", 1); - } - - break; - } - case search::ParseItem::ITEM_PREFIXTERM: - case search::ParseItem::ITEM_SUBSTRINGTERM: - case search::ParseItem::ITEM_EXACTSTRINGTERM: - case search::ParseItem::ITEM_NUMTERM: - case search::ParseItem::ITEM_TERM: - if (!IsLegalIndex(si.getIndexName())) - continue; - { - // add a new keyword - vespalib::stringref term = si.getTerm(); - if ( !term.empty() && useful(creator)) { - // An actual string to add - keywords.append(term.data(), term.size()); - keywords.append("\0", 1); - } - } - break; - - default: - // Do nothing to AND, RANK, OR - break; - } - } - iteratorloopend: - // Add a 'blank' keyword - keywords.append("\0", 1); - - // Must now allocate a string and copy the data from the rawbuf - void *result = malloc(keywords.GetUsedLen()); - if (result != nullptr) { - memcpy(result, keywords.GetDrainPos(), keywords.GetUsedLen()); - } - return static_cast<char *>(result); -} - } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h index 11e8da1ac7b..5f87de762f9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/searchlib/util/rawbuf.h> namespace search::docsummary { @@ -17,7 +16,7 @@ public: { vespalib::string _prefix; public: - explicit IndexPrefix(const char *prefix); + explicit IndexPrefix(const char *prefix) noexcept; ~IndexPrefix(); bool Match(const char *idxName) const; const vespalib::string& get_prefix() const noexcept { return _prefix; } @@ -25,12 +24,11 @@ public: private: typedef vespalib::hash_set<vespalib::string> Set; - IDocsumEnvironment *_env; - std::vector<IndexPrefix> _legalPrefixes; - Set _legalIndexes; + const IDocsumEnvironment *_env; + std::vector<IndexPrefix> _legalPrefixes; + Set _legalIndexes; - bool IsLegalIndexPrefix(const char *idxName) const - { + bool IsLegalIndexPrefix(const char *idxName) const { for (auto& prefix : _legalPrefixes ) { if (prefix.Match(idxName)) { return true; @@ -39,34 +37,19 @@ private: return false; } - bool IsLegalIndexName(const char *idxName) const; -public: - explicit KeywordExtractor(IDocsumEnvironment * env); - KeywordExtractor(const KeywordExtractor &) = delete; - KeywordExtractor& operator=(const KeywordExtractor &) = delete; - ~KeywordExtractor(); - - - /** - * Add a prefix to the set of legal index name prefixes. - * - * @param prefix the index name prefix to add. - **/ - void AddLegalIndexPrefix(const char *prefix) - { + void AddLegalIndexPrefix(const char *prefix) { _legalPrefixes.emplace_back(prefix); } - - /** - * Add a name to the set of legal index names. - * - * @param idxName the index name to add. - **/ - void AddLegalIndexName(const char *idxName) - { + void AddLegalIndexName(const char *idxName) { _legalIndexes.insert(idxName); } + bool IsLegalIndexName(const char *idxName) const; +public: + explicit KeywordExtractor(const IDocsumEnvironment * env); + KeywordExtractor(const KeywordExtractor &) = delete; + KeywordExtractor& operator=(const KeywordExtractor &) = delete; + ~KeywordExtractor(); /** @@ -98,27 +81,6 @@ public: * @return true if the given index name is legal. **/ bool IsLegalIndex(vespalib::stringref idx) const; - - - /** - * Extract keywords from a stack dump of a SimpleQueryStack. - * - * The words are extracted as follows: For AND and OR operators, all - * TERM items occuring in a legal index (the set of legal indexes is - * defined by invoking the @ref AddLegalIndex and @ref - * AddLegalIndexPrefix methods) are extracted. - * - * For PHRASE operators, the TERMS in a phrase are put together with - * space between them. - * - * @todo For NOT operators, only the first operand is considered. - * - * @param buf Pointer to buffer with simple query stack dump. - * @param bufLen Length of stack dump buffer - * @return Pointer to a buffer containing zero-terminated keywords, - * with an empty word at the end. - */ - char *ExtractKeywords(vespalib::stringref buf) const; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index aa7db0e2745..1000fee8423 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -252,7 +252,7 @@ PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType typ //-------------------------------------------------------------------------- -PositionsDFW::UP PositionsDFW::create(const char *attribute_name, IAttributeManager *attribute_manager, bool useV8geoPositions) { +PositionsDFW::UP PositionsDFW::create(const char *attribute_name, const IAttributeManager *attribute_manager, bool useV8geoPositions) { PositionsDFW::UP ret; if (attribute_manager != nullptr) { if (!attribute_name) { @@ -274,7 +274,7 @@ PositionsDFW::UP PositionsDFW::create(const char *attribute_name, IAttributeMana } std::unique_ptr<DocsumFieldWriter> -AbsDistanceDFW::create(const char *attribute_name, IAttributeManager *attribute_manager) { +AbsDistanceDFW::create(const char *attribute_name, const IAttributeManager *attribute_manager) { std::unique_ptr<DocsumFieldWriter> ret; if (attribute_manager != nullptr) { if (!attribute_name) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index c0d9d7d8111..d0a38d1004e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -46,7 +46,7 @@ public: void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; - static std::unique_ptr<DocsumFieldWriter> create(const char *attribute_name, IAttributeManager *index_man); + static std::unique_ptr<DocsumFieldWriter> create(const char *attribute_name, const IAttributeManager *index_man); }; @@ -61,7 +61,7 @@ public: PositionsDFW(const vespalib::string & attrName, bool useV8geoPositions); bool IsGenerated() const override { return true; } void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; - static UP create(const char *attribute_name, IAttributeManager *index_man, bool useV8geoPositions); + static UP create(const char *attribute_name, const IAttributeManager *index_man, bool useV8geoPositions); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index 5b08020c010..b7b10d9c1ea 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -7,9 +7,7 @@ namespace search::docsummary { -RankFeaturesDFW::RankFeaturesDFW(IDocsumEnvironment * env) : - _env(env) -{ } +RankFeaturesDFW::RankFeaturesDFW() = default; RankFeaturesDFW::~RankFeaturesDFW() = default; @@ -18,7 +16,7 @@ RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) const { if ( !state->_rankFeatures ) { - state->_callback.FillRankFeatures(state, _env); + state->_callback.FillRankFeatures(*state); if (state->_rankFeatures.get() == nullptr) { // still no rank features to write return; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h index d0516f9f0b7..7302d162b65 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h @@ -6,15 +6,10 @@ namespace search::docsummary { -class IDocsumEnvironment; - class RankFeaturesDFW : public SimpleDFW { -private: - IDocsumEnvironment * _env; - public: - RankFeaturesDFW(IDocsumEnvironment * env); + RankFeaturesDFW(); RankFeaturesDFW(const RankFeaturesDFW &) = delete; RankFeaturesDFW & operator=(const RankFeaturesDFW &) = delete; ~RankFeaturesDFW() override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp new file mode 100644 index 00000000000..668fb9b519b --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "res_config_entry.h" + +namespace search::docsummary { + +ResConfigEntry::ResConfigEntry() noexcept + : _type(RES_BAD), + _bindname(), + _enumValue(0) +{ +} + +ResConfigEntry::~ResConfigEntry() = default; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h new file mode 100644 index 00000000000..771125b1f45 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_config_entry.h @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "res_type.h" +#include <vespa/vespalib/stllike/string.h> + +namespace search::docsummary { + +/** + * This struct describes a single docsum field (name and type). + **/ +struct ResConfigEntry { + ResType _type; + vespalib::string _bindname; + int _enumValue; + ResConfigEntry() noexcept; + ~ResConfigEntry(); +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type.h b/searchsummary/src/vespa/searchsummary/docsummary/res_type.h index 02c9f1522a4..e86e0eeedc8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/res_type.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type.h @@ -24,7 +24,8 @@ enum ResType { RES_LONG_DATA, RES_JSONSTRING, RES_TENSOR, - RES_FEATUREDATA + RES_FEATUREDATA, + RES_BAD }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp index 98cc8372ac1..f7a1aae7455 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.cpp @@ -22,8 +22,63 @@ ResTypeUtils::GetResTypeName(ResType type) case RES_JSONSTRING: return "jsonstring"; case RES_TENSOR: return "tensor"; case RES_FEATUREDATA: return "featuredata"; + default: return "unknown-type"; } - return "unknown-type"; +} + +ResType +ResTypeUtils::get_res_type(vespalib::stringref name) +{ + if (name == "integer") { + return RES_INT; + } + if (name == "short") { + return RES_SHORT; + } + if (name == "byte") { + return RES_BYTE; + } + if (name == "bool") { + return RES_BOOL; + } + if (name == "float") { + return RES_FLOAT; + } + if (name == "double") { + return RES_DOUBLE; + } + if (name == "int64") { + return RES_INT64; + } + if (name == "string") { + return RES_STRING; + } + if (name == "data") { + return RES_DATA; + } + if (name == "longstring") { + return RES_LONG_STRING; + } + if (name == "longdata") { + return RES_LONG_DATA; + } + if (name == "jsonstring") { + return RES_JSONSTRING; + } + if (name == "tensor") { + return RES_TENSOR; + } + if (name == "featuredata") { + return RES_FEATUREDATA; + } + // Known aliases + if (name == "raw") { + return RES_DATA; + } + if (name == "xmlstring") { + return RES_JSONSTRING; + } + return RES_BAD; } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h index 9d73b7dc3fd..a2e881b5b4d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/res_type_utils.h @@ -3,6 +3,7 @@ #pragma once #include "res_type.h" +#include <vespa/vespalib/stllike/string.h> namespace search::docsummary { @@ -16,6 +17,8 @@ struct ResTypeUtils * @param resType enum value of a result field type. **/ static const char *GetResTypeName(ResType type); + + static ResType get_res_type(vespalib::stringref name); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 5df9ebebdf0..7fb94f48ac3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -2,8 +2,7 @@ #pragma once -#include "res_type.h" -#include <vespa/searchlib/util/rawbuf.h> +#include "res_config_entry.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/searchlib/util/stringenum.h> @@ -11,16 +10,6 @@ namespace search::docsummary { /** - * This struct describes a single docsum field (name and type). - **/ -struct ResConfigEntry { - ResType _type; - vespalib::string _bindname; - int _enumValue; -}; - - -/** * This class represents a specific docsum format (docsum class). It * contains an array of ResConfigEntry instances (config * entries). It also contains methods for mapping both diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 85d197f799a..f5e4c5d34cf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -129,39 +129,10 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const for (unsigned int j = 0; rc && (j < cfg_class.fields.size()); j++) { const char *fieldtype = cfg_class.fields[j].type.c_str(); const char *fieldname = cfg_class.fields[j].name.c_str(); + auto res_type = ResTypeUtils::get_res_type(fieldtype); LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg_class.name.c_str(), fieldname, fieldtype); - if (strcmp(fieldtype, "integer") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_INT); - } else if (strcmp(fieldtype, "short") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_SHORT); - } else if (strcmp(fieldtype, "bool") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_BOOL); - } else if (strcmp(fieldtype, "byte") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_BYTE); - } else if (strcmp(fieldtype, "float") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_FLOAT); - } else if (strcmp(fieldtype, "double") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_DOUBLE); - } else if (strcmp(fieldtype, "int64") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_INT64); - } else if (strcmp(fieldtype, "string") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_STRING); - } else if (strcmp(fieldtype, "data") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_DATA); - } else if (strcmp(fieldtype, "raw") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_DATA); - } else if (strcmp(fieldtype, "longstring") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_LONG_STRING); - } else if (strcmp(fieldtype, "longdata") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_LONG_DATA); - } else if (strcmp(fieldtype, "xmlstring") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_JSONSTRING); - } else if (strcmp(fieldtype, "jsonstring") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_JSONSTRING); - } else if (strcmp(fieldtype, "tensor") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_TENSOR); - } else if (strcmp(fieldtype, "featuredata") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_FEATUREDATA); + if (res_type != RES_BAD) { + rc = resClass->AddConfigEntry(fieldname, res_type); } else { LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg_class.name.c_str(), j, fieldtype); rc = false; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index f5fc942e40c..76bae0cee97 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -11,10 +11,7 @@ LOG_SETUP(".searchlib.docsummary.summaryfeaturesdfw"); namespace search::docsummary { -SummaryFeaturesDFW::SummaryFeaturesDFW(IDocsumEnvironment * env) : - _env(env) -{ -} +SummaryFeaturesDFW::SummaryFeaturesDFW() = default; SummaryFeaturesDFW::~SummaryFeaturesDFW() = default; @@ -27,7 +24,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, return; } if ( ! state->_summaryFeatures) { - state->_callback.FillSummaryFeatures(state, _env); + state->_callback.FillSummaryFeatures(*state); if ( !state->_summaryFeatures) { // still no summary features to write return; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h index 45d7f7fa641..ec14dc45055 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h @@ -6,15 +6,10 @@ namespace search::docsummary { -class IDocsumEnvironment; - class SummaryFeaturesDFW : public SimpleDFW { -private: - IDocsumEnvironment * _env; - public: - SummaryFeaturesDFW(IDocsumEnvironment * env); + SummaryFeaturesDFW(); SummaryFeaturesDFW(const SummaryFeaturesDFW &) = delete; SummaryFeaturesDFW & operator=(const SummaryFeaturesDFW &) = delete; ~SummaryFeaturesDFW() override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 218a9b82803..8bf78b90c77 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -35,8 +35,8 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/util/exceptions.h> @@ -152,11 +152,11 @@ void handleIndexingTerms(Handler &handler, const StringFieldValue &value) { } } sort(terms.begin(), terms.end()); - SpanTermVector::const_iterator it = terms.begin(); - SpanTermVector::const_iterator ite = terms.end(); + auto it = terms.begin(); + auto ite = terms.end(); int32_t endPos = 0; for (; it != ite; ) { - SpanTermVector::const_iterator it_begin = it; + auto it_begin = it; if (it_begin->first.from() > endPos) { Span tmpSpan(endPos, it_begin->first.from() - endPos); handler.handleAnnotations(tmpSpan, it, it); @@ -584,10 +584,10 @@ public: SlimeInserter inserter(slime); SlimeFiller visitor(inserter, _tokenize, _matching_elems); input.accept(visitor); - search::RawBuf rbuf(4_Ki); - search::SlimeOutputRawBufAdapter adapter(rbuf); - vespalib::slime::BinaryFormat::encode(slime, adapter); - return std::make_unique<RawFieldValue>(rbuf.GetDrainPos(), rbuf.GetUsedLen()); + vespalib::SmartBuffer buffer(4_Ki); + vespalib::slime::BinaryFormat::encode(slime, buffer); + vespalib::Memory mem = buffer.obtain(); + return std::make_unique<RawFieldValue>(mem.data, mem.size); } }; diff --git a/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h index df035fca685..71c2be19bba 100644 --- a/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h +++ b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h @@ -16,8 +16,8 @@ public: { } ~MockStateCallback() override { } - void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } - void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } + void FillSummaryFeatures(GetDocsumsState&) override { } + void FillRankFeatures(GetDocsumsState&) override { } std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields&) override { return std::make_unique<MatchingElements>(_matching_elems); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java index ae6cef65156..240ec40adbf 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java @@ -117,7 +117,6 @@ public class TransportSecurityUtils { SystemTlsContext(Path tlsOptionsConfigFile) { super(tlsOptionsConfigFile, getInsecureAuthorizationMode()); } - - @Override public void close() { throw new UnsupportedOperationException("Shared TLS context cannot be closed"); } + @Override public void close() {} } } diff --git a/slobrok/src/Doxyfile b/slobrok/src/Doxyfile deleted file mode 100644 index 10727c87e68..00000000000 --- a/slobrok/src/Doxyfile +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.4.7 - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- -PROJECT_NAME = slobrok -PROJECT_NUMBER = -OUTPUT_DIRECTORY = ../doc/doxygen -CREATE_SUBDIRS = NO -OUTPUT_LANGUAGE = English -USE_WINDOWS_ENCODING = NO -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = NO -STRIP_FROM_PATH = -STRIP_FROM_INC_PATH = -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -DETAILS_AT_TOP = NO -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 8 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -BUILTIN_STL_SUPPORT = NO -DISTRIBUTE_GROUP_DOC = NO -SUBGROUPING = YES -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- -EXTRACT_ALL = YES -EXTRACT_PRIVATE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = NO -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = YES -HIDE_SCOPE_NAMES = NO -SHOW_INCLUDE_FILES = YES -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_BY_SCOPE_NAME = YES -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_DIRECTORIES = YES -FILE_VERSION_FILTER = -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_NO_PARAMDOC = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LOGFILE = -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- -INPUT = -FILE_PATTERNS = -RECURSIVE = YES -EXCLUDE = regress -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = tstdst* \ - sbcmd* \ - check_slobrok* -EXAMPLE_PATH = -EXAMPLE_PATTERNS = -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- -SOURCE_BROWSER = NO -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = YES -REFERENCES_RELATION = YES -REFERENCES_LINK_SOURCE = YES -USE_HTAGS = NO -VERBATIM_HEADERS = YES -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- -ALPHABETICAL_INDEX = NO -COLS_IN_ALPHA_INDEX = 5 -IGNORE_PREFIX = -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_STYLESHEET = -HTML_ALIGN_MEMBERS = YES -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -BINARY_TOC = NO -TOC_EXPAND = NO -DISABLE_INDEX = NO -ENUM_VALUES_PER_LINE = 4 -GENERATE_TREEVIEW = NO -TREEVIEW_WIDTH = 250 -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- -GENERATE_LATEX = YES -LATEX_OUTPUT = latex -LATEX_CMD_NAME = latex -MAKEINDEX_CMD_NAME = makeindex -COMPACT_LATEX = NO -PAPER_TYPE = a4wide -EXTRA_PACKAGES = -LATEX_HEADER = -PDF_HYPERLINKS = NO -USE_PDFLATEX = NO -LATEX_BATCHMODE = NO -LATEX_HIDE_INDICES = NO -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- -GENERATE_RTF = NO -RTF_OUTPUT = rtf -COMPACT_RTF = NO -RTF_HYPERLINKS = NO -RTF_STYLESHEET_FILE = -RTF_EXTENSIONS_FILE = -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- -GENERATE_MAN = NO -MAN_OUTPUT = man -MAN_EXTENSION = .3 -MAN_LINKS = NO -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- -GENERATE_XML = NO -XML_OUTPUT = xml -XML_SCHEMA = -XML_DTD = -XML_PROGRAMLISTING = YES -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- -GENERATE_AUTOGEN_DEF = NO -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- -GENERATE_PERLMOD = NO -PERLMOD_LATEX = NO -PERLMOD_PRETTY = YES -PERLMOD_MAKEVAR_PREFIX = -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = NO -EXPAND_ONLY_PREDEF = NO -SEARCH_INCLUDES = YES -INCLUDE_PATH = -INCLUDE_FILE_PATTERNS = -PREDEFINED = -EXPAND_AS_DEFINED = -SKIP_FUNCTION_MACROS = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- -TAGFILES = ../../vespalib/doc/doxygen/vespalib.tag=../vespalib -GENERATE_TAGFILE = ../doc/doxygen/slobrok.tag -ALLEXTERNALS = NO -EXTERNAL_GROUPS = YES -PERL_PATH = /usr/bin/perl -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- -CLASS_DIAGRAMS = YES -HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = YES -CLASS_GRAPH = YES -COLLABORATION_GRAPH = YES -GROUP_GRAPHS = YES -UML_LOOK = NO -TEMPLATE_RELATIONS = NO -INCLUDE_GRAPH = YES -INCLUDED_BY_GRAPH = YES -CALL_GRAPH = YES -CALLER_GRAPH = NO -GRAPHICAL_HIERARCHY = YES -DIRECTORY_GRAPH = YES -DOT_IMAGE_FORMAT = png -DOT_PATH = -DOTFILE_DIRS = -MAX_DOT_GRAPH_WIDTH = 1024 -MAX_DOT_GRAPH_HEIGHT = 1024 -MAX_DOT_GRAPH_DEPTH = 0 -DOT_TRANSPARENT = NO -DOT_MULTI_TARGETS = NO -GENERATE_LEGEND = YES -DOT_CLEANUP = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- -SEARCHENGINE = NO diff --git a/storage/src/Doxyfile b/storage/src/Doxyfile deleted file mode 100644 index 0ae14d0acc1..00000000000 --- a/storage/src/Doxyfile +++ /dev/null @@ -1,994 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.2.18 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# General configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = Storage - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../doc - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, -# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en -# (Japanese with english messages), Korean, Norwegian, Polish, Portuguese, -# Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish and Ukrainian. - -OUTPUT_LANGUAGE = English - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these class will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower case letters. If set to YES upper case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are adviced to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explict @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# reimplements. - -INHERIT_DOCS = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consist of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = YES - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = storage - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl - -FILE_PATTERNS = *.h *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet - -HTML_STYLESHEET = ../cpp/vespa_link.css - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output dir. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non empty doxygen will try to run -# the html help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the Html help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript and frames is required (for instance Mozilla, Netscape 4.0+, -# or Internet explorer 4.0+). Note that for large projects the tree generation -# can take a very long time. In such cases it is better to disable this feature. -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimised for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assigments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_XML = NO - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_PREDEF_ONLY tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES tag can be used to specify one or more tagfiles. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in Html, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superceded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yield more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermedate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO - -# The CGI_NAME tag should be the name of the CGI script that -# starts the search engine (doxysearch) with the correct parameters. -# A script with this name will be generated by doxygen. - -CGI_NAME = search.cgi - -# The CGI_URL tag should be the absolute URL to the directory where the -# cgi binaries are located. See the documentation of your http daemon for -# details. - -CGI_URL = - -# The DOC_URL tag should be the absolute URL to the directory where the -# documentation is located. If left blank the absolute path to the -# documentation, with file:// prepended to it, will be used. - -DOC_URL = - -# The DOC_ABSPATH tag should be the absolute path to the directory where the -# documentation is located. If left blank the directory on the local machine -# will be used. - -DOC_ABSPATH = - -# The BIN_ABSPATH tag must point to the directory where the doxysearch binary -# is installed. - -BIN_ABSPATH = /usr/local/bin/ - -# The EXT_DOC_PATHS tag can be used to specify one or more paths to -# documentation generated for other projects. This allows doxysearch to search -# the documentation for these projects as well. - -EXT_DOC_PATHS = diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 4c1b1662f68..a5f5075a4d8 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1966,6 +1966,7 @@ TEST_F(FileStorManagerTest, bucket_db_is_populated_from_provider_when_initialize getDummyPersistence().set_fake_bucket_set(buckets); c.manager->initialize_bucket_databases_from_provider(); + c.manager->complete_internal_initialization(); std::vector<std::pair<document::BucketId, api::BucketInfo>> from_db; auto populate_from_db = [&from_db](uint64_t key, auto& entry) { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index 124acf1864c..8bd892e0f09 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -103,6 +103,8 @@ public: /** Get info for given bucket (Used for whitebox testing) */ StorBucketDatabase::Entry getBucketInfo(const document::Bucket &id) const; + void force_db_sweep_and_metric_update() { updateMetrics(true); } + private: friend struct BucketManagerTest; diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h index 3da6bd3151a..337c3ea7ff0 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.h +++ b/storage/src/vespa/storage/common/statusmetricconsumer.h @@ -34,6 +34,10 @@ public: const std::string& name = "status"); ~StatusMetricConsumer() override; + // Metric reporting requires the "vespa.content.metrics_api" capability + CapabilitySet required_capabilities() const noexcept override { + return CapabilitySet::of({ Capability::content_metrics_api() }); + } vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream& out, const framework::HttpUrlPath&) const override; void updateMetrics(const MetricLockGuard & guard) override; diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index a9b7a727a5b..7139ab0eb41 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/component/vtag.h> +#include <vespa/vespalib/net/connection_auth_context.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/config/subscription/configuri.h> #include <vespa/config/helper/configfetcher.hpp> @@ -45,15 +46,15 @@ StatusWebServer::~StatusWebServer() void StatusWebServer::configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config) { int newPort = config->httpport; - // If server is already running, ignore config updates that doesn't - // alter port, or suggests random port. + // If server is already running, ignore config updates that doesn't + // alter port, or suggests random port. if (_httpServer) { if (newPort == 0 || newPort == _port) return; } - // Try to create new server before destroying old. + // Try to create new server before destroying old. LOG(info, "Starting status web server on port %u.", newPort); std::unique_ptr<WebServer> server; - // Negative port number means don't run the web server + // Negative port number means don't run the web server if (newPort >= 0) { try { server = std::make_unique<WebServer>(*this, newPort); @@ -146,6 +147,25 @@ StatusWebServer::getListenPort() const } void +StatusWebServer::invoke_reporter(const framework::StatusReporter& reporter, + const framework::HttpUrlPath& url_path, + vespalib::Portal::GetRequest& request) +{ + try { + std::ostringstream content; + auto content_type = reporter.getReportContentType(url_path); + if (reporter.reportStatus(content, url_path)) { + request.respond_with_content(content_type, content.str()); + } else { + request.respond_with_error(404, "Not Found"); + } + } catch (std::exception &e) { + LOG(warning, "Internal Server Error: %s", e.what()); + request.respond_with_error(500, "Internal Server Error"); + } +} + +void StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, vespalib::Portal::GetRequest request) { vespalib::string link(urlpath.getPath()); @@ -157,22 +177,25 @@ StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, vespalib::Por if ( ! link.empty()) { const framework::StatusReporter *reporter = _reporterMap.getStatusReporter(link); if (reporter != nullptr) { - try { - std::ostringstream content; - auto content_type = reporter->getReportContentType(urlpath); - if (reporter->reportStatus(content, urlpath)) { - request.respond_with_content(content_type, content.str()); - } else { - request.respond_with_error(404, "Not Found"); - } - } catch (std::exception &e) { - LOG(warning, "Internal Server Error: %s", e.what()); - request.respond_with_error(500, "Internal Server Error"); + const auto& auth_ctx = request.auth_context(); + if (auth_ctx.capabilities().contains_all(reporter->required_capabilities())) { + invoke_reporter(*reporter, urlpath, request); + } else { + // TODO should print peer address as well; not currently exposed + LOG(warning, "Peer with %s denied status page access to '%s' due to insufficient " + "credentials (had %s, needed %s)", + auth_ctx.peer_credentials().to_string().c_str(), + link.c_str(), auth_ctx.capabilities().to_string().c_str(), + reporter->required_capabilities().to_string().c_str()); + request.respond_with_error(403, "Forbidden"); } } else { request.respond_with_error(404, "Not Found"); } } else { + // TODO should the index page be capability-restricted? Would be a bit strange if the root + // index '/' page requires status capabilities but '/metrics' does not. + // The index page only leaks the Vespa version and node type (inferred by reporter set). IndexPageReporter indexRep; indexRep << "<p><b>Binary version of Vespa:</b> " << vespalib::Vtag::currentVersion.toString() diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h index 429f5249441..26db7ff5069 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h @@ -24,6 +24,7 @@ namespace config { namespace storage { namespace framework { + struct StatusReporter; struct StatusReporterMap; struct ThreadHandle; struct ComponentRegister; @@ -35,10 +36,10 @@ namespace framework { class StatusWebServer : private config::IFetcherCallback<vespa::config::content::core::StorStatusConfig> { class WebServer : public vespalib::Portal::GetHandler { - StatusWebServer& _status; - vespalib::Portal::SP _server; + StatusWebServer& _status; + vespalib::Portal::SP _server; vespalib::ThreadStackExecutor _executor; - vespalib::Portal::Token::UP _root; + vespalib::Portal::Token::UP _root; public: WebServer(StatusWebServer&, uint16_t port); @@ -81,6 +82,9 @@ public: int getListenPort() const; void handlePage(const framework::HttpUrlPath&, vespalib::Portal::GetRequest request); private: + void invoke_reporter(const framework::StatusReporter&, + const framework::HttpUrlPath&, + vespalib::Portal::GetRequest&); void configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config) override; }; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index c7ee0ab97e0..63fec9f037f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -1041,7 +1041,9 @@ void FileStorManager::initialize_bucket_databases_from_provider() { const double elapsed = start_time.getElapsedTimeAsDouble(); LOG(info, "Completed listing of %zu buckets in %.2g milliseconds", bucket_count, elapsed); _metrics->bucket_db_init_latency.addValue(elapsed); +} +void FileStorManager::complete_internal_initialization() { update_reported_state_after_db_init(); _init_handler.notifyDoneInitializing(); } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 83f1826c498..6820ed6f451 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -110,7 +110,12 @@ public: // By ensuring that this function is called prior to chain opening, this invariant // shall be upheld since no RPC/MessageBus endpoints have been made available // yet at that point in time. + // Must always be called _before_ complete_internal_initialization() void initialize_bucket_databases_from_provider(); + // Tag node internally as having completed initialization. Updates reported state + // (although this will not be communicated out of the process until the + // CommunicationManager thread has been fired up). + void complete_internal_initialization(); const FileStorMetrics& get_metrics() const { return *_metrics; } diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 1cb2334803e..b76e2cca02a 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -33,6 +33,7 @@ ServiceLayerNode::ServiceLayerNode(const config::ConfigUri & configUri, ServiceL _context(context), _persistenceProvider(persistenceProvider), _externalVisitors(externalVisitors), + _bucket_manager(nullptr), _fileStorManager(nullptr), _init_has_been_called(false) { @@ -160,7 +161,9 @@ ServiceLayerNode::createChain(IStorageChainBuilder &builder) auto merge_throttler = merge_throttler_up.get(); builder.add(std::move(merge_throttler_up)); builder.add(std::make_unique<ChangedBucketOwnershipHandler>(_configUri, compReg)); - builder.add(std::make_unique<BucketManager>(_configUri, _context.getComponentRegister())); + auto bucket_manager = std::make_unique<BucketManager>(_configUri, _context.getComponentRegister()); + _bucket_manager = bucket_manager.get(); + builder.add(std::move(bucket_manager)); builder.add(std::make_unique<VisitorManager>(_configUri, _context.getComponentRegister(), static_cast<VisitorMessageSessionFactory &>(*this), _externalVisitors)); builder.add(std::make_unique<ModifiedBucketChecker>( _context.getComponentRegister(), _persistenceProvider, _configUri)); @@ -186,7 +189,20 @@ ServiceLayerNode::pause() void ServiceLayerNode::perform_post_chain_creation_init_steps() { assert(_fileStorManager); + assert(_bucket_manager); + // After initialization, the node will immediately start communicating with the cluster + // controller, exchanging host info. This host info contains a subset snapshot of the active + // metrics, which includes the total bucket count, doc count etc. It is critical that + // we must never report back host info _prior_ to having run at least one full sweep of + // the bucket database, lest we risk transiently reporting zero buckets held by the + // content node. Doing so could cause orchestration logic to perform operations based + // on erroneous assumptions. + // To avoid this, we explicitly force a full DB sweep and metric update prior to reporting + // the node as up. Since this function is called prior to the CommunicationManager thread + // being started, any CC health pings should also always happen after this init step. _fileStorManager->initialize_bucket_databases_from_provider(); + _bucket_manager->force_db_sweep_and_metric_update(); + _fileStorManager->complete_internal_initialization(); } } // storage diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index f0189f943ab..4b0e0f73408 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -18,6 +18,7 @@ namespace storage { namespace spi { struct PersistenceProvider; } +class BucketManager; class FileStorManager; class ServiceLayerNode @@ -31,6 +32,7 @@ class ServiceLayerNode // FIXME: Should probably use the fetcher in StorageNode std::unique_ptr<config::ConfigFetcher> _configFetcher; + BucketManager* _bucket_manager; FileStorManager* _fileStorManager; bool _init_has_been_called; diff --git a/storage/src/vespa/storageframework/generic/status/statusreporter.h b/storage/src/vespa/storageframework/generic/status/statusreporter.h index cc48bb841fd..3f84d5e8ae4 100644 --- a/storage/src/vespa/storageframework/generic/status/statusreporter.h +++ b/storage/src/vespa/storageframework/generic/status/statusreporter.h @@ -16,12 +16,15 @@ #include <ostream> #include <vespa/storageframework/generic/status/httpurlpath.h> +#include <vespa/vespalib/net/tls/capability_set.h> #include <vespa/vespalib/stllike/string.h> namespace storage::framework { -struct StatusReporter -{ +struct StatusReporter { + using Capability = vespalib::net::tls::Capability; + using CapabilitySet = vespalib::net::tls::CapabilitySet; + StatusReporter(vespalib::stringref id, vespalib::stringref name); virtual ~StatusReporter(); @@ -40,6 +43,16 @@ struct StatusReporter virtual bool isValidStatusRequest() const { return true; } /** + * By default, a status reporter requires the "vespa.content.status_pages" client capability. + * This can be overridden by subclasses to require reporter-specific capabilities + * (or none at all). If the client does not satisfy the required capabilities, a + * "403 Forbidden" error response will be returned to the client. + */ + virtual CapabilitySet required_capabilities() const noexcept { + return CapabilitySet::of({ Capability::content_status_pages() }); + } + + /** * Called to get content type. * An empty string indicates page not found. */ diff --git a/streamingvisitors/src/tests/docsum/docsum.cpp b/streamingvisitors/src/tests/docsum/docsum.cpp index 475489d2f5a..b7f45123c48 100644 --- a/streamingvisitors/src/tests/docsum/docsum.cpp +++ b/streamingvisitors/src/tests/docsum/docsum.cpp @@ -7,6 +7,8 @@ #include <vespa/vsm/common/docsum.h> #include <vespa/vsm/vsm/flattendocsumwriter.h> #include <vespa/vsm/vsm/slimefieldwriter.h> +#include <vespa/vespalib/data/smart_buffer.h> +#include <vespa/vespalib/data/slime/slime.h> using namespace document; @@ -30,18 +32,18 @@ private: public: TestDocument(const search::DocumentIdT & docId, size_t numFields) : vsm::Document(docId, numFields), _fields(numFields) {} - virtual bool setField(FieldIdT fId, document::FieldValue::UP fv) override { + bool setField(FieldIdT fId, document::FieldValue::UP fv) override { if (fId < _fields.size()) { _fields[fId].reset(fv.release()); return true; } return false; } - virtual const document::FieldValue * getField(FieldIdT fId) const override { + const document::FieldValue * getField(FieldIdT fId) const override { if (fId < _fields.size()) { return _fields[fId].get(); } - return NULL; + return nullptr; } }; @@ -105,12 +107,22 @@ DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValu } void +convert(SlimeFieldWriter & sfw, const document::FieldValue & fv, vespalib::Output & output) +{ + vespalib::Slime slime; + vespalib::slime::SlimeInserter inserter(slime); + sfw.insert(fv, inserter); + vespalib::slime::BinaryFormat::encode(slime, output); +} + +void DocsumTest::assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp) { - sfw.convert(fv); + vespalib::SmartBuffer buffer(1024); + convert(sfw, fv, buffer); vespalib::Slime gotSlime; - vespalib::Memory serialized(sfw.out()); + vespalib::Memory serialized(buffer.obtain()); size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized, gotSlime); ASSERT_EQUAL(decodeRes, serialized.size); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index c2bbf4d09da..4973db909e4 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -3,6 +3,7 @@ #include "querytermdata.h" #include "searchenvironment.h" #include "searchvisitor.h" +#include "matching_elements_filler.h" #include <vespa/persistence/spi/docentry.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> @@ -17,8 +18,8 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/data/slime/slime.h> #include <vespa/fnet/databuffer.h> -#include "matching_elements_filler.h" #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchvisitor"); @@ -92,7 +93,7 @@ createMultiValueAttribute(const vespalib::string & name, const document::FieldVa LOG(debug, "Can not make an multivalue attribute out of %s with data type '%s' (%s)", name.c_str(), ndt->getName().c_str(), fv.className()); } - return AttributeVector::SP(); + return {}; } AttributeVector::SP @@ -108,7 +109,7 @@ createAttribute(const vespalib::string & name, const document::FieldValue & fv) } else { LOG(debug, "Can not make an attribute out of %s of type '%s'.", name.c_str(), fv.className()); } - return AttributeVector::SP(); + return {}; } SearchVisitor::SummaryGenerator::SummaryGenerator() : @@ -117,7 +118,7 @@ SearchVisitor::SummaryGenerator::SummaryGenerator() : _docsumState(_callback), _docsumFilter(), _docsumWriter(nullptr), - _rawBuf(4_Ki) + _buf(4_Ki) { } @@ -128,12 +129,20 @@ vespalib::ConstBufferRef SearchVisitor::SummaryGenerator::fillSummary(AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) { if (_docsumWriter != nullptr) { - _rawBuf.reset(); _docsumState._args.setResultClassName(summaryClass); - uint32_t docsumLen = _docsumWriter->WriteDocsum(lid, &_docsumState, _docsumFilter.get(), &_rawBuf); - return vespalib::ConstBufferRef(_rawBuf.GetDrainPos(), docsumLen); + vespalib::Slime slime; + vespalib::slime::SlimeInserter inserter(slime); + _docsumWriter->WriteDocsum(lid, &_docsumState, _docsumFilter.get(), inserter); + + _buf.reset(); + vespalib::WritableMemory magicId = _buf.reserve(4); + memcpy(magicId.data, &search::docsummary::SLIME_MAGIC_ID, 4); + _buf.commit(4); + vespalib::slime::BinaryFormat::encode(slime, _buf); + vespalib::Memory mem = _buf.obtain(); + return {mem.data, mem.size}; } - return vespalib::ConstBufferRef(); + return {}; } void SearchVisitor::HitsResultPreparator::execute(vespalib::Identifiable & obj) @@ -612,10 +621,10 @@ SearchVisitor::registerAdditionalFields(const std::vector<vsm::DocsumTools::Fiel for (const vsm::DocsumTools::FieldSpec & spec : docsumSpec) { fieldList.push_back(spec.getOutputName()); const std::vector<vespalib::string> & inputNames = spec.getInputNames(); - for (size_t j = 0; j < inputNames.size(); ++j) { - fieldList.push_back(inputNames[j]); - if (PositionDataType::isZCurveFieldName(inputNames[j])) { - fieldList.emplace_back(PositionDataType::cutZCurveFieldName(inputNames[j])); + for (const auto & name : inputNames) { + fieldList.push_back(name); + if (PositionDataType::isZCurveFieldName(name)) { + fieldList.emplace_back(PositionDataType::cutZCurveFieldName(name)); } } } @@ -694,25 +703,9 @@ SearchVisitor::setupDocsumObjects() for (const IAttributeVector * v : ds->_attributes) { if (v != nullptr) { vespalib::string name(v->getName()); - vsm::FieldIdT fid = _fieldSearchSpecMap.nameIdMap().fieldNo(name); - if ( fid != StringFieldIdTMap::npos ) { - AttributeGuard::UP attr(_attrMan.getAttribute(name)); - if (attr->valid()) { - size_t index(_attributeFields.size()); - for (size_t j(0); j < index; j++) { - if (_attributeFields[j]._field == fid) { - index = j; - } - } - if (index == _attributeFields.size()) { - _attributeFields.emplace_back(fid, std::move(attr)); - } - } else { - LOG(warning, "Attribute '%s' is not valid", name.c_str()); - } - } else { - LOG(warning, "No field with name '%s'. Odd ....", name.c_str()); - } + auto msg = vespalib::make_string("Illegal config: Docsum field writer using attribute vector '%s' configured for streaming search", name.c_str()); + LOG(error, "%s", msg.c_str()); + throw vespalib::IllegalStateException(msg); } } } else { @@ -732,7 +725,7 @@ SearchVisitor::setupAttributeVectors() void SearchVisitor::setupAttributeVector(const FieldPath &fieldPath) { vespalib::string attrName(fieldPath.front().getName()); - for (FieldPath::const_iterator ft(fieldPath.begin() + 1), fmt(fieldPath.end()); ft != fmt; ft++) { + for (auto ft(fieldPath.begin() + 1), fmt(fieldPath.end()); ft != fmt; ft++) { attrName.append("."); attrName.append((*ft)->getName()); } @@ -855,7 +848,7 @@ private: bool SearchVisitor::compatibleDocumentTypes(const document::DocumentType& typeA, - const document::DocumentType& typeB) const + const document::DocumentType& typeB) { if (&typeA == &typeB) { return true; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 20ab1ccf325..98aa6b89c9c 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -18,6 +18,7 @@ #include <vespa/vsm/vsm/vsm-adapter.h> #include <vespa/vespalib/objects/objectoperation.h> #include <vespa/vespalib/objects/objectpredicate.h> +#include <vespa/vespalib/data/smart_buffer.h> #include <vespa/searchlib/query/streaming/query.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/attribute/attributemanager.h> @@ -44,7 +45,7 @@ public: SearchVisitor(storage::StorageComponent&, storage::VisitorEnvironment& vEnv, const vdslib::Parameters & params); - ~SearchVisitor(); + ~SearchVisitor() override; private: /** * This struct wraps an attribute vector. @@ -101,7 +102,7 @@ private: class PositionInserter : public AttributeInserter { public: PositionInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId); - ~PositionInserter(); + ~PositionInserter() override; private: void onPrimitive(uint32_t fid, const Content & c) override; void onStructStart(const Content & fv) override; @@ -128,9 +129,9 @@ private: /** * Process attribute hints and add needed attributes to the given list. **/ - void processHintedAttributes(const IndexEnvironment & indexEnv, bool rank, - const search::IAttributeManager & attrMan, - std::vector<AttrInfo> & attributeFields); + static void processHintedAttributes(const IndexEnvironment & indexEnv, bool rank, + const search::IAttributeManager & attrMan, + std::vector<AttrInfo> & attributeFields); public: RankController(); @@ -244,8 +245,8 @@ private: * @param docsumSpec config with the field names used by the docsum setup. * @param fieldList list of field names that are built. **/ - void registerAdditionalFields(const std::vector<vsm::DocsumTools::FieldSpec> & docsumSpec, - std::vector<vespalib::string> & fieldList); + static void registerAdditionalFields(const std::vector<vsm::DocsumTools::FieldSpec> & docsumSpec, + std::vector<vespalib::string> & fieldList); /** * Setup the field searchers used when matching the query with the stream of documents. @@ -301,8 +302,8 @@ private: DocEntryList& entries, HitCounter& hitCounter) override; - bool compatibleDocumentTypes(const document::DocumentType& typeA, - const document::DocumentType& typeB) const; + static bool compatibleDocumentTypes(const document::DocumentType& typeA, + const document::DocumentType& typeB); /** * Process one document @@ -369,7 +370,7 @@ private: class GroupingEntry : std::shared_ptr<Grouping> { public: - GroupingEntry(Grouping * grouping); + explicit GroupingEntry(Grouping * grouping); ~GroupingEntry(); void aggregate(const document::Document & doc, search::HitRank rank); const Grouping & operator * () const { return *_grouping; } @@ -387,32 +388,32 @@ private: { public: SummaryGenerator(); - ~SummaryGenerator(); + ~SummaryGenerator() override; GetDocsumsState & getDocsumState() { return _docsumState; } vsm::GetDocsumsStateCallback & getDocsumCallback() { return _callback; } void setFilter(std::unique_ptr<vsm::DocsumFilter> filter) { _docsumFilter = std::move(filter); } void setDocsumCache(const vsm::IDocSumCache & cache) { _docsumFilter->setDocSumStore(cache); } void setDocsumWriter(IDocsumWriter & docsumWriter) { _docsumWriter = & docsumWriter; } - virtual vespalib::ConstBufferRef fillSummary(search::AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) override; + vespalib::ConstBufferRef fillSummary(search::AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) override; private: vsm::GetDocsumsStateCallback _callback; GetDocsumsState _docsumState; std::unique_ptr<vsm::DocsumFilter> _docsumFilter; search::docsummary::IDocsumWriter * _docsumWriter; - search::RawBuf _rawBuf; + vespalib::SmartBuffer _buf; }; class HitsResultPreparator : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: - HitsResultPreparator(SummaryGenerator & summaryGenerator) : + explicit HitsResultPreparator(SummaryGenerator & summaryGenerator) : _summaryGenerator(summaryGenerator), _numHitsAggregators(0) { } size_t getNumHitsAggregators() const { return _numHitsAggregators; } private: - virtual void execute(vespalib::Identifiable &obj) override; - virtual bool check(const vespalib::Identifiable &obj) const override; + void execute(vespalib::Identifiable &obj) override; + bool check(const vespalib::Identifiable &obj) const override; SummaryGenerator & _summaryGenerator; size_t _numHitsAggregators; }; @@ -461,7 +462,7 @@ class SearchVisitorFactory : public storage::VisitorFactory { storage::Visitor* makeVisitor(storage::StorageComponent&, storage::VisitorEnvironment&env, const vdslib::Parameters& params) override; public: - SearchVisitorFactory(const config::ConfigUri & configUri); + explicit SearchVisitorFactory(const config::ConfigUri & configUri); }; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/CMakeLists.txt b/streamingvisitors/src/vespa/vsm/vsm/CMakeLists.txt index adc00b341a3..6afb61078c7 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/CMakeLists.txt +++ b/streamingvisitors/src/vespa/vsm/vsm/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(vsm_vsmbase OBJECT SOURCES docsumfieldspec.cpp docsumfilter.cpp + docsum_field_writer_factory.cpp fieldsearchspec.cpp flattendocsumwriter.cpp slimefieldwriter.cpp diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp new file mode 100644 index 00000000000..35410f3ec67 --- /dev/null +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsum_field_writer_factory.h" +#include <vespa/searchsummary/docsummary/copy_dfw.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer.h> +#include <vespa/searchsummary/docsummary/empty_dfw.h> +#include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> +#include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/vsm/config/config-vsmfields.h> + +using search::MatchingElementsFields; +using search::docsummary::CopyDFW; +using search::docsummary::EmptyDFW; +using search::docsummary::IDocsumEnvironment; +using search::docsummary::DocsumFieldWriter; +using search::docsummary::MatchedElementsFilterDFW; +using vespa::config::search::vsm::VsmfieldsConfig; + +namespace vsm { + +namespace { + +void populate_fields(MatchingElementsFields& fields, VsmfieldsConfig& fields_config, const vespalib::string& field_name) +{ + vespalib::string prefix = field_name + "."; + for (const auto& spec : fields_config.fieldspec) { + if (spec.name.substr(0, prefix.size()) == prefix) { + fields.add_mapping(field_name, spec.name); + } + if (spec.name == field_name) { + fields.add_field(field_name); + } + } +} + +} + +DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env, const vespa::config::search::vsm::VsmfieldsConfig& vsm_fields_config) + : search::docsummary::DocsumFieldWriterFactory(use_v8_geo_positions, env), + _vsm_fields_config(vsm_fields_config) +{ +} + +DocsumFieldWriterFactory::~DocsumFieldWriterFactory() = default; + +std::unique_ptr<DocsumFieldWriter> +DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) +{ + std::unique_ptr<DocsumFieldWriter> fieldWriter; + if ((overrideName == "staticrank") || + (overrideName == "ranklog") || + (overrideName == "label") || + (overrideName == "project") || + (overrideName == "positions") || + (overrideName == "absdist") || + (overrideName == "subproject")) + { + fieldWriter = std::make_unique<EmptyDFW>(); + rc = true; + } else if ((overrideName == "attribute") || + (overrideName == "attributecombiner")) { + if (!argument.empty() && argument != fieldName) { + fieldWriter = std::make_unique<CopyDFW>(argument); + } + rc = true; + } else if (overrideName == "geopos") { + rc = true; + } else if ((overrideName == "matchedattributeelementsfilter") || + (overrideName == "matchedelementsfilter")) { + vespalib::string source_field = argument.empty() ? fieldName : argument; + populate_fields(*_matching_elems_fields, _vsm_fields_config, source_field); + fieldWriter = MatchedElementsFilterDFW::create(source_field, _matching_elems_fields); + rc = static_cast<bool>(fieldWriter); + } else { + return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(fieldName, overrideName, argument, rc); + } + return fieldWriter; +} + +} diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h new file mode 100644 index 00000000000..c06cb0454b3 --- /dev/null +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchsummary/docsummary/docsum_field_writer_factory.h> +#include <vespa/vsm/config/config-vsmfields.h> + +namespace vsm { + +/* + * Factory interface class for creating docsum field writers, adjusted for + * streaming search. + */ +class DocsumFieldWriterFactory : public search::docsummary::DocsumFieldWriterFactory +{ + const vespa::config::search::vsm::VsmfieldsConfig& _vsm_fields_config; + +public: + DocsumFieldWriterFactory(bool use_v8_geo_positions, const search::docsummary::IDocsumEnvironment& env, const vespa::config::search::vsm::VsmfieldsConfig& vsm_fields_config); + ~DocsumFieldWriterFactory() override; + std::unique_ptr<search::docsummary::DocsumFieldWriter> + create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) override; +}; + +} diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp index 5b9c73cb07d..59e56e657be 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp @@ -1,80 +1,24 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vsm/vsm/docsumconfig.h> -#include <vespa/searchsummary/docsummary/copy_dfw.h> -#include <vespa/searchsummary/docsummary/empty_dfw.h> -#include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> +#include "docsumconfig.h" #include <vespa/searchsummary/docsummary/resultconfig.h> -#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vsm/config/config-vsmfields.h> -#include <vespa/vsm/config/config-vsmsummary.h> +#include "docsum_field_writer_factory.h" -using search::MatchingElementsFields; -using search::docsummary::DocsumFieldWriter; -using search::docsummary::CopyDFW; -using search::docsummary::EmptyDFW; -using search::docsummary::MatchedElementsFilterDFW; -using search::docsummary::ResultConfig; using vespa::config::search::vsm::VsmfieldsConfig; -using vespa::config::search::vsm::VsmsummaryConfig; namespace vsm { -namespace { - -void populate_fields(MatchingElementsFields& fields, VsmfieldsConfig& fields_config, const vespalib::string& field_name) -{ - vespalib::string prefix = field_name + "."; - for (const auto& spec : fields_config.fieldspec) { - if (spec.name.substr(0, prefix.size()) == prefix) { - fields.add_mapping(field_name, spec.name); - } - if (spec.name == field_name) { - fields.add_field(field_name); - } - } -} - -} - -DynamicDocsumConfig::DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config) +DynamicDocsumConfig::DynamicDocsumConfig(const search::docsummary::IDocsumEnvironment& env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config) : Parent(env, writer), _vsm_fields_config(std::move(vsm_fields_config)) { } -std::unique_ptr<DocsumFieldWriter> -DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) +std::unique_ptr<search::docsummary::IDocsumFieldWriterFactory> +DynamicDocsumConfig::make_docsum_field_writer_factory() { - std::unique_ptr<DocsumFieldWriter> fieldWriter; - if ((overrideName == "staticrank") || - (overrideName == "ranklog") || - (overrideName == "label") || - (overrideName == "project") || - (overrideName == "positions") || - (overrideName == "absdist") || - (overrideName == "subproject")) - { - fieldWriter = std::make_unique<EmptyDFW>(); - rc = true; - } else if ((overrideName == "attribute") || - (overrideName == "attributecombiner")) { - if (!argument.empty() && argument != fieldName) { - fieldWriter = std::make_unique<CopyDFW>(argument); - } - rc = true; - } else if (overrideName == "geopos") { - rc = true; - } else if ((overrideName == "matchedattributeelementsfilter") || - (overrideName == "matchedelementsfilter")) { - string source_field = argument.empty() ? fieldName : argument; - populate_fields(*matching_elems_fields, *_vsm_fields_config, source_field); - fieldWriter = MatchedElementsFilterDFW::create(source_field, matching_elems_fields); - rc = static_cast<bool>(fieldWriter); - } else { - fieldWriter = search::docsummary::DynamicDocsumConfig::createFieldWriter(fieldName, overrideName, argument, rc, matching_elems_fields); - } - return fieldWriter; + return std::make_unique<DocsumFieldWriterFactory>(getResultConfig().useV8geoPositions(), getEnvironment(), *_vsm_fields_config); } } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h index a660c544d7d..760d493dbc1 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h @@ -18,11 +18,9 @@ public: private: std::shared_ptr<VsmfieldsConfig> _vsm_fields_config; public: - DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config); + DynamicDocsumConfig(const search::docsummary::IDocsumEnvironment& env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config); private: - std::unique_ptr<search::docsummary::DocsumFieldWriter> - createFieldWriter(const string & fieldName, const string & overrideName, - const string & cf, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override; + std::unique_ptr<search::docsummary::IDocsumFieldWriterFactory> make_docsum_field_writer_factory() override; }; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 8bd416ca716..a49eec1a869 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -7,6 +7,7 @@ #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.docsumfilter"); diff --git a/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.cpp b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.cpp index 5f0ec60656e..f0278bb3470 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slimefieldwriter.h" -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/document/datatype/positiondatatype.h> +#include <vespa/vespalib/data/slime/slime.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.slimefieldwriter"); @@ -13,19 +13,6 @@ LOG_SETUP(".vsm.slimefieldwriter"); namespace { vespalib::string -toString(const vsm::FieldPath & fieldPath) -{ - vespalib::asciistream oss; - for (size_t i = 0; i < fieldPath.size(); ++i) { - if (i > 0) { - oss << "."; - } - oss << fieldPath[i].getName(); - } - return oss.str(); -} - -vespalib::string toString(const std::vector<vespalib::string> & fieldPath) { vespalib::asciistream oss; @@ -42,7 +29,6 @@ toString(const std::vector<vespalib::string> & fieldPath) using namespace vespalib::slime::convenience; - namespace vsm { void @@ -52,18 +38,17 @@ SlimeFieldWriter::traverseRecursive(const document::FieldValue & fv, Inserter &i fv.className(), fv.toString().c_str(), toString(_currPath).c_str()); if (fv.isCollection()) { - const document::CollectionFieldValue & cfv = static_cast<const document::CollectionFieldValue &>(fv); + const auto & cfv = static_cast<const document::CollectionFieldValue &>(fv); if (cfv.isA(document::FieldValue::Type::ARRAY)) { - const document::ArrayFieldValue & afv = static_cast<const document::ArrayFieldValue &>(cfv); + const auto & afv = static_cast<const document::ArrayFieldValue &>(cfv); Cursor &a = inserter.insertArray(); - for (size_t i = 0; i < afv.size(); ++i) { - const document::FieldValue & nfv = afv[i]; + for (const auto & nfv : afv) { ArrayInserter ai(a); traverseRecursive(nfv, ai); } } else { assert(cfv.isA(document::FieldValue::Type::WSET)); - const document::WeightedSetFieldValue & wsfv = static_cast<const document::WeightedSetFieldValue &>(cfv); + const auto & wsfv = static_cast<const document::WeightedSetFieldValue &>(cfv); Cursor &a = inserter.insertArray(); Symbol isym = a.resolve("item"); Symbol wsym = a.resolve("weight"); @@ -77,7 +62,7 @@ SlimeFieldWriter::traverseRecursive(const document::FieldValue & fv, Inserter &i } } } else if (fv.isA(document::FieldValue::Type::MAP)) { - const document::MapFieldValue & mfv = static_cast<const document::MapFieldValue &>(fv); + const auto & mfv = static_cast<const document::MapFieldValue &>(fv); Cursor &a = inserter.insertArray(); Symbol keysym = a.resolve("key"); Symbol valsym = a.resolve("value"); @@ -85,13 +70,13 @@ SlimeFieldWriter::traverseRecursive(const document::FieldValue & fv, Inserter &i Cursor &o = a.addObject(); ObjectSymbolInserter ki(o, keysym); traverseRecursive(*entry.first, ki); - _currPath.push_back("value"); + _currPath.emplace_back("value"); ObjectSymbolInserter vi(o, valsym); traverseRecursive(*entry.second, vi); _currPath.pop_back(); } } else if (fv.isStructured()) { - const document::StructuredFieldValue & sfv = static_cast<const document::StructuredFieldValue &>(fv); + const auto & sfv = static_cast<const document::StructuredFieldValue &>(fv); Cursor &o = inserter.insertObject(); if (sfv.getDataType() == &document::PositionDataType::getInstance() && search::docsummary::ResultConfig::wantedV8geoPositions()) @@ -134,7 +119,7 @@ SlimeFieldWriter::traverseRecursive(const document::FieldValue & fv, Inserter &i } } else { if (fv.isLiteral()) { - const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv); + const auto & lfv = static_cast<const document::LiteralFieldValueB &>(fv); inserter.insertString(lfv.getValueRef()); } else if (fv.isNumeric()) { switch (fv.getDataType()->getId()) { @@ -169,8 +154,8 @@ SlimeFieldWriter::explorePath(vespalib::stringref candidate) return true; } // find out if we should explore the current path - for (size_t i = 0; i < _inputFields->size(); ++i) { - const FieldPath & fp = (*_inputFields)[i].getPath(); + for (const auto & field : *_inputFields) { + const FieldPath & fp = field.getPath(); if (_currPath.size() <= fp.size()) { bool equal = true; for (size_t j = 0; j < _currPath.size() && equal; ++j) { @@ -190,8 +175,6 @@ SlimeFieldWriter::explorePath(vespalib::stringref candidate) } SlimeFieldWriter::SlimeFieldWriter() : - _rbuf(4_Ki), - _slime(), _inputFields(nullptr), _currPath() { @@ -205,22 +188,4 @@ SlimeFieldWriter::insert(const document::FieldValue & fv, vespalib::slime::Inser traverseRecursive(fv, inserter); } -void -SlimeFieldWriter::convert(const document::FieldValue & fv) -{ - if (LOG_WOULD_LOG(debug)) { - if (_inputFields != nullptr) { - for (size_t i = 0; i < _inputFields->size(); ++i) { - LOG(debug, "write: input field path [%zd] '%s'", i, toString((*_inputFields)[i].getPath()).c_str()); - } - } else { - LOG(debug, "write: no input fields"); - } - } - SlimeInserter inserter(_slime); - traverseRecursive(fv, inserter); - search::SlimeOutputRawBufAdapter adapter(_rbuf); - vespalib::slime::BinaryFormat::encode(_slime, adapter); -} - } diff --git a/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h index 3d7ece93694..0907783feaf 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h @@ -4,8 +4,8 @@ #include "docsumfieldspec.h" #include <vespa/vsm/common/storagedocument.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/searchlib/util/rawbuf.h> + +namespace vespalib::slime { class Inserter; } namespace vsm { @@ -17,8 +17,6 @@ namespace vsm { class SlimeFieldWriter { private: - search::RawBuf _rbuf; - vespalib::Slime _slime; const DocsumFieldSpec::FieldIdentifierVector * _inputFields; std::vector<vespalib::string> _currPath; @@ -29,7 +27,6 @@ public: SlimeFieldWriter(); ~SlimeFieldWriter(); - /** * Specifies the subset of the field value that should be written. **/ @@ -40,20 +37,7 @@ public: **/ void insert(const document::FieldValue & fv, vespalib::slime::Inserter& inserter); - /** - * Convert the given field value - **/ - void convert(const document::FieldValue & fv); - - /** - * Return a reference to the output binary data - **/ - vespalib::stringref out() const { - return vespalib::stringref(_rbuf.GetDrainPos(), _rbuf.GetUsedLen()); - } - void clear() { - _rbuf.Reuse(); _inputFields = nullptr; _currPath.clear(); } diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index e7532d2a25a..2bf6b2f3972 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -4,6 +4,7 @@ #include "docsumconfig.h" #include "i_matching_elements_filler.h" #include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.vsm-adapter"); @@ -21,20 +22,18 @@ GetDocsumsStateCallback::GetDocsumsStateCallback() : _matching_elements_filler() { } -void GetDocsumsStateCallback::FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) +void GetDocsumsStateCallback::FillSummaryFeatures(GetDocsumsState& state) { - (void) env; if (_summaryFeatures) { // set the summary features to write to the docsum - state->_summaryFeatures = _summaryFeatures; - state->_summaryFeaturesCached = true; + state._summaryFeatures = _summaryFeatures; + state._summaryFeaturesCached = true; } } -void GetDocsumsStateCallback::FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) +void GetDocsumsStateCallback::FillRankFeatures(GetDocsumsState& state) { - (void) env; if (_rankFeatures) { // set the rank features to write to the docsum - state->_rankFeatures = _rankFeatures; + state._rankFeatures = _rankFeatures; } } @@ -69,16 +68,24 @@ DocsumTools::FieldSpec::FieldSpec() : DocsumTools::FieldSpec::~FieldSpec() = default; -DocsumTools::DocsumTools(std::unique_ptr<DynamicDocsumWriter> writer) : - _writer(std::move(writer)), - _juniper(), - _resultClass(), - _fieldSpecs() -{ } +DocsumTools::DocsumTools() + : IDocsumEnvironment(), + _writer(), + _juniper(), + _resultClass(), + _fieldSpecs() +{ +} DocsumTools::~DocsumTools() = default; +void +DocsumTools::set_writer(std::unique_ptr<DynamicDocsumWriter> writer) +{ + _writer = std::move(writer); +} + bool DocsumTools::obtainFieldNames(const FastS_VsmsummaryHandle &cfg) { @@ -134,6 +141,14 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) LOG(debug, "configureVsmSummary(): Size of cfg fieldmap: %zd", vsmSummary->fieldmap.size()); // UlfC: debugging LOG(debug, "configureVsmSummary(): outputclass='%s'", vsmSummary->outputclass.c_str()); // UlfC: debugging + // create new docsum tools + auto docsumTools = std::make_unique<DocsumTools>(); + + // configure juniper (used when configuring DynamicDocsumConfig) + _juniperProps = std::make_unique<JuniperProperties>(*juniperrc); + auto juniper = std::make_unique<juniper::Juniper>(_juniperProps.get(), &_wordFolder); + docsumTools->setJuniper(std::move(juniper)); + // init result config auto resCfg = std::make_unique<ResultConfig>(); if ( ! resCfg->ReadConfig(*summary.get(), _configId.c_str())) { @@ -148,17 +163,10 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) // create dynamic docsum writer auto writer = std::make_unique<DynamicDocsumWriter>(std::move(resCfg), std::move(kwExtractor)); - - // configure juniper (used when configuring DynamicDocsumConfig) - _juniperProps = std::make_unique<JuniperProperties>(*juniperrc); - auto juniper = std::make_unique<juniper::Juniper>(_juniperProps.get(), &_wordFolder); - - // create new docsum tools - auto docsumTools = std::make_unique<DocsumTools>(std::move(writer)); - docsumTools->setJuniper(std::move(juniper)); + docsumTools->set_writer(std::move(writer)); // configure dynamic docsum writer - DynamicDocsumConfig dynDocsumConfig(docsumTools.get(), docsumTools->getDocsumWriter(), _fieldsCfg.get()); + DynamicDocsumConfig dynDocsumConfig(*docsumTools, docsumTools->getDocsumWriter(), _fieldsCfg.get()); dynDocsumConfig.configure(*summaryMap.get()); // configure new docsum tools diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index 656ab2e8fc6..d6c1f55d092 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -38,8 +38,8 @@ private: public: GetDocsumsStateCallback(); - void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; - void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; + void FillSummaryFeatures(GetDocsumsState& state) override; + void FillRankFeatures(GetDocsumsState& state) override; virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env); virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } @@ -77,8 +77,9 @@ private: DocsumTools &operator=(const DocsumTools &); public: - DocsumTools(std::unique_ptr<DynamicDocsumWriter> writer); + DocsumTools(); ~DocsumTools(); + void set_writer(std::unique_ptr<DynamicDocsumWriter> writer); void setJuniper(std::unique_ptr<juniper::Juniper> juniper) { _juniper = std::move(juniper); } const ResultConfig *getResultConfig() const { return _writer->GetResultConfig(); } DynamicDocsumWriter *getDocsumWriter() const { return _writer.get(); } @@ -87,9 +88,9 @@ public: bool obtainFieldNames(const FastS_VsmsummaryHandle &cfg); // inherit doc from IDocsumEnvironment - search::IAttributeManager * getAttributeManager() override { return NULL; } + const search::IAttributeManager * getAttributeManager() const override { return nullptr; } vespalib::string lookupIndex(const vespalib::string&) const override { return ""; } - juniper::Juniper * getJuniper() override { return _juniper.get(); } + const juniper::Juniper * getJuniper() const override { return _juniper.get(); } }; typedef std::shared_ptr<DocsumTools> DocsumToolsPtr; diff --git a/vbench/src/vbench/core/Doxyfile b/vbench/src/vbench/core/Doxyfile deleted file mode 100644 index 2b7a0a133ff..00000000000 --- a/vbench/src/vbench/core/Doxyfile +++ /dev/null @@ -1,1162 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.3.9.1 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = vbench_core - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of source -# files, where putting all generated files in the same directory would otherwise -# cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, -# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, -# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, -# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, -# Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is used -# as the annotated text. Otherwise, the brief description is used as-is. If left -# blank, the following values are used ("$name" is automatically replaced with the -# name of the entity): "The $name class" "The $name widget" "The $name file" -# "is" "provides" "specifies" "contains" "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explicit @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. - -SHOW_DIRECTORIES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm - -FILE_PATTERNS = - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = NO - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superseded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = YES - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 4096 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 4096 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes that -# lay further from the root node will be omitted. Note that setting this option to -# 1 or 2 may greatly reduce the computation time needed for large code bases. Also -# note that a graph may be further truncated if the graph's image dimensions are -# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). -# If 0 is used for the depth value (the default), the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO diff --git a/vespa-maven-plugin/pom.xml b/vespa-maven-plugin/pom.xml index c7ea54a3977..e5b7dec71e4 100644 --- a/vespa-maven-plugin/pom.xml +++ b/vespa-maven-plugin/pom.xml @@ -26,6 +26,11 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> </dependency> diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java index 4e5c80e1099..8e51555457b 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/CompileVersionMojo.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.plugin; +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.text.XML; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; @@ -30,14 +32,17 @@ public class CompileVersionMojo extends AbstractVespaMojo { protected void doExecute() throws IOException { Path output = Paths.get(outputFile).toAbsolutePath(); OptionalInt allowMajor = majorVersion(new File(project.getBasedir(), "src/main/application/deployment.xml").toPath()); - String compileVersion = controller.compileVersion(id, allowMajor); - if (allowMajor.isPresent()) { - getLog().info("Allowing major version " + allowMajor.getAsInt() + "."); - } - getLog().info("Vespa version to compile against is '" + compileVersion + "'."); + allowMajor.ifPresent(major -> getLog().info("Allowing only major version " + major + ".")); + + Version compileVersion = Version.fromString(controller.compileVersion(id, allowMajor)); + if (compileVersion.isAfter(Vtag.currentVersion)) + throw new IllegalStateException("parent version (" + Vtag.currentVersion.toFullString() + ") should be at least as " + + "high as the Vespa version to compile against (" + compileVersion.toFullString() + ")"); + + getLog().info("Vespa version to compile against is '" + compileVersion.toFullString() + "'."); getLog().info("Writing compile version to '" + output + "'."); Files.createDirectories(output.getParent()); - Files.writeString(output, compileVersion); + Files.writeString(output, compileVersion.toFullString()); } /** Returns the major version declared in given deploymentXml, if any */ diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index afed1595a19..628ebe6b074 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -35,26 +35,7 @@ consider_fallback () { read_conf_file () { deffile="$VESPA_HOME/conf/vespa/default-env.txt" if [ -f "${deffile}" ]; then - eval $(perl -ne ' - chomp; - my ($action, $varname, $value) = split(" ", $_, 3); - $varname =~ s{[.]}{__}g; - if ($varname !~ m{^\w+}) { - # print STDERR "invalid var name $varname" - next; - } - if ($action eq "fallback" || $action eq "override") { - next if ($action eq "fallback" && $ENV{$varname} ne ""); - # quote value - if ($value !~ m{^["][^"]*["]$} ) { - $value =~ s{(\W)}{\\$1}g; - } - print "$varname=$value; export $varname; " - } - if ($action eq "unset") { - print "export -n $varname; unset $varname; " - } - ' < $deffile) + eval $(${VESPA_HOME}/libexec/vespa/script-utils export-env) fi } @@ -329,8 +310,7 @@ use_configserver_if_needed () { } getJavaOptionsIPV46() { - canon_ipv4=$(hostname | perl -pe 'chomp; ($_,$rest) = gethostbyname($_);') - if [ -z "${canon_ipv4}" ]; then + if ${VESPA_HOME}/libexec/vespa/script-utils ipv6-only; then echo " -Djava.net.preferIPv6Addresses=true" fi } diff --git a/vespalib/src/Doxyfile b/vespalib/src/Doxyfile deleted file mode 100644 index 0d63f28039d..00000000000 --- a/vespalib/src/Doxyfile +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -DOXYFILE_ENCODING = UTF-8 -PROJECT_NAME = vespalib -PROJECT_NUMBER = -OUTPUT_DIRECTORY = ../doc/doxygen -CREATE_SUBDIRS = NO -OUTPUT_LANGUAGE = English -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = -STRIP_FROM_INC_PATH = ./ -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = YES -QT_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 8 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -OPTIMIZE_FOR_FORTRAN = NO -OPTIMIZE_OUTPUT_VHDL = NO -EXTENSION_MAPPING = -BUILTIN_STL_SUPPORT = NO -CPP_CLI_SUPPORT = NO -SIP_SUPPORT = NO -IDL_PROPERTY_SUPPORT = YES -DISTRIBUTE_GROUP_DOC = NO -SUBGROUPING = YES -TYPEDEF_HIDES_STRUCT = NO -SYMBOL_CACHE_SIZE = 0 -EXTRACT_ALL = YES -EXTRACT_PRIVATE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -EXTRACT_ANON_NSPACES = NO -HIDE_UNDOC_MEMBERS = YES -HIDE_UNDOC_CLASSES = YES -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = YES -HIDE_SCOPE_NAMES = NO -SHOW_INCLUDE_FILES = YES -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_MEMBERS_CTORS_1ST = NO -SORT_GROUP_NAMES = NO -SORT_BY_SCOPE_NAME = NO -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_DIRECTORIES = NO -SHOW_FILES = YES -SHOW_NAMESPACES = YES -FILE_VERSION_FILTER = -LAYOUT_FILE = -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_NO_PARAMDOC = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LOGFILE = -INPUT = vespalib/util \ - vespalib/component \ - vespalib/cppunit \ - vespalib/geo \ - vespalib/io \ - vespalib/text \ - vespalib/testkit -INPUT_ENCODING = UTF-8 -FILE_PATTERNS = -RECURSIVE = NO -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = -EXCLUDE_SYMBOLS = -EXAMPLE_PATH = -EXAMPLE_PATTERNS = -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -SOURCE_BROWSER = YES -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = NO -REFERENCES_RELATION = NO -REFERENCES_LINK_SOURCE = YES -USE_HTAGS = NO -VERBATIM_HEADERS = YES -ALPHABETICAL_INDEX = YES -COLS_IN_ALPHA_INDEX = 5 -IGNORE_PREFIX = -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_TIMESTAMP = NO -HTML_STYLESHEET = -HTML_ALIGN_MEMBERS = YES -HTML_DYNAMIC_SECTIONS = NO -GENERATE_DOCSET = NO -DOCSET_FEEDNAME = "Doxygen generated docs" -DOCSET_BUNDLE_ID = org.doxygen.Project -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -CHM_INDEX_ENCODING = -BINARY_TOC = NO -TOC_EXPAND = NO -GENERATE_QHP = NO -QCH_FILE = -QHP_NAMESPACE = -QHP_VIRTUAL_FOLDER = doc -QHP_CUST_FILTER_NAME = -QHP_CUST_FILTER_ATTRS = -QHP_SECT_FILTER_ATTRS = -QHG_LOCATION = -DISABLE_INDEX = NO -ENUM_VALUES_PER_LINE = 4 -GENERATE_TREEVIEW = NO -USE_INLINE_TREES = NO -TREEVIEW_WIDTH = 250 -FORMULA_FONTSIZE = 10 -SEARCHENGINE = NO -GENERATE_LATEX = NO -LATEX_OUTPUT = latex -LATEX_CMD_NAME = latex -MAKEINDEX_CMD_NAME = makeindex -COMPACT_LATEX = NO -PAPER_TYPE = a4wide -EXTRA_PACKAGES = -LATEX_HEADER = -PDF_HYPERLINKS = NO -USE_PDFLATEX = NO -LATEX_BATCHMODE = NO -LATEX_HIDE_INDICES = NO -LATEX_SOURCE_CODE = NO -GENERATE_RTF = NO -RTF_OUTPUT = rtf -COMPACT_RTF = NO -RTF_HYPERLINKS = NO -RTF_STYLESHEET_FILE = -RTF_EXTENSIONS_FILE = -GENERATE_MAN = NO -MAN_OUTPUT = man -MAN_EXTENSION = .3 -MAN_LINKS = NO -GENERATE_XML = NO -XML_OUTPUT = xml -XML_SCHEMA = -XML_DTD = -XML_PROGRAMLISTING = YES -GENERATE_AUTOGEN_DEF = NO -GENERATE_PERLMOD = NO -PERLMOD_LATEX = NO -PERLMOD_PRETTY = YES -PERLMOD_MAKEVAR_PREFIX = -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = NO -EXPAND_ONLY_PREDEF = NO -SEARCH_INCLUDES = YES -INCLUDE_PATH = -INCLUDE_FILE_PATTERNS = -PREDEFINED = IAM_DOXYGEN -EXPAND_AS_DEFINED = -SKIP_FUNCTION_MACROS = YES -TAGFILES = -GENERATE_TAGFILE = ../doc/doxygen/vespalib.tag -ALLEXTERNALS = NO -EXTERNAL_GROUPS = YES -PERL_PATH = /usr/bin/perl -CLASS_DIAGRAMS = YES -MSCGEN_PATH = -HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = YES -DOT_FONTNAME = FreeSans -DOT_FONTSIZE = 10 -DOT_FONTPATH = -CLASS_GRAPH = NO -COLLABORATION_GRAPH = YES -GROUP_GRAPHS = YES -UML_LOOK = NO -TEMPLATE_RELATIONS = NO -INCLUDE_GRAPH = NO -INCLUDED_BY_GRAPH = NO -CALL_GRAPH = NO -CALLER_GRAPH = NO -GRAPHICAL_HIERARCHY = YES -DIRECTORY_GRAPH = YES -DOT_IMAGE_FORMAT = png -DOT_PATH = -DOTFILE_DIRS = -DOT_GRAPH_MAX_NODES = 50 -MAX_DOT_GRAPH_DEPTH = 0 -DOT_TRANSPARENT = NO -DOT_MULTI_TARGETS = NO -GENERATE_LEGEND = YES -DOT_CLEANUP = YES diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 3d19c335c19..0178443643e 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -622,8 +622,8 @@ TEST_F("Peer credentials are propagated to CryptoCodec", CertFixture) { auto& client_creds = f.server->peer_credentials(); auto& server_creds = f.client->peer_credentials(); - fprintf(stderr, "Client credentials (observed by server): %s\n", to_string(client_creds).c_str()); - fprintf(stderr, "Server credentials (observed by client): %s\n", to_string(server_creds).c_str()); + fprintf(stderr, "Client credentials (observed by server): %s\n", client_creds.to_string().c_str()); + fprintf(stderr, "Server credentials (observed by client): %s\n", server_creds.to_string().c_str()); EXPECT_EQUAL("rockets.wile.example.com", client_creds.common_name); ASSERT_EQUAL(2u, client_creds.dns_sans.size()); diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index 2334f5d64b5..db6baf2ef4a 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -3,9 +3,9 @@ #pragma once #include "type.h" -#include <vespa/vespalib/data/memory.h> #include "symbol.h" #include "external_memory.h" +#include <vespa/vespalib/data/memory.h> namespace vespalib { class Slime; } diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp index de079261d6d..8b7bed62ac5 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp @@ -32,8 +32,7 @@ SmartBuffer::drop() { alloc::Alloc empty_buf; _data.swap(empty_buf); - _read_pos = 0; - _write_pos = 0; + reset(); } SmartBuffer::SmartBuffer(size_t initial_size) diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.h b/vespalib/src/vespa/vespalib/data/smart_buffer.h index eb817e71bbf..17fb7614f0e 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.h +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.h @@ -39,6 +39,10 @@ public: drop(); } } + void reset() { + _read_pos = 0; + _write_pos = 0; + } Memory obtain() override; Input &evict(size_t bytes) override; WritableMemory reserve(size_t bytes) override; diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp index d7977f6cd2a..e088eeb4906 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp @@ -482,7 +482,7 @@ bool OpenSslTlsContextImpl::verify_trusted_certificate(::X509_STORE_CTX* store_c // Buffer warnings on peer IP address to avoid log flooding. LOGBT(warning, codec_impl.peer_address().ip_address(), "Certificate verification of peer '%s' failed with %s", - codec_impl.peer_address().spec().c_str(), to_string(creds).c_str()); + codec_impl.peer_address().spec().c_str(), creds.to_string().c_str()); return (authz_mode != AuthorizationMode::Enforce); } // Store away credentials and role set for later use by requests that arrive over this connection. diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp index 9a001e24fea..92854bdd7d5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.cpp @@ -14,7 +14,7 @@ PeerCredentials& PeerCredentials::operator=(PeerCredentials&&) noexcept = defaul PeerCredentials::~PeerCredentials() = default; std::ostream& operator<<(std::ostream& os, const PeerCredentials& creds) { - os << to_string(creds); + os << creds.to_string(); return os; } @@ -36,20 +36,20 @@ void emit_comma_separated_string_list(asciistream& os, stringref title, } } -vespalib::string to_string(const PeerCredentials& creds) { +vespalib::string PeerCredentials::to_string() const { asciistream os; os << "PeerCredentials("; bool emit_comma = false; - if (!creds.common_name.empty()) { - os << "CN '" << creds.common_name << "'"; + if (!common_name.empty()) { + os << "CN '" << common_name << "'"; emit_comma = true; } - if (!creds.dns_sans.empty()) { - emit_comma_separated_string_list(os, "DNS SANs", creds.dns_sans, emit_comma); + if (!dns_sans.empty()) { + emit_comma_separated_string_list(os, "DNS SANs", dns_sans, emit_comma); emit_comma = true; } - if (!creds.uri_sans.empty()) { - emit_comma_separated_string_list(os, "URI SANs", creds.uri_sans, emit_comma); + if (!uri_sans.empty()) { + emit_comma_separated_string_list(os, "URI SANs", uri_sans, emit_comma); } os << ')'; return os.str(); diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h index b81772d2bce..22c98c023b5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h +++ b/vespalib/src/vespa/vespalib/net/tls/peer_credentials.h @@ -23,10 +23,10 @@ struct PeerCredentials { PeerCredentials(PeerCredentials&&) noexcept; PeerCredentials& operator=(PeerCredentials&&) noexcept; ~PeerCredentials(); + + vespalib::string to_string() const; }; std::ostream& operator<<(std::ostream&, const PeerCredentials&); -vespalib::string to_string(const PeerCredentials&); - } diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index b5b1641c54e..e88493c1a1b 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) add_subdirectory(zookeeper-server) -add_subdirectory(zookeeper-server-3.8.0) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index cf19f15e71f..69a38ef9e2a 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -14,7 +14,6 @@ <modules> <module>zookeeper-server-common</module> <module>zookeeper-server</module> - <module>zookeeper-server-3.8.0</module> </modules> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.8.0/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.8.0/CMakeLists.txt deleted file mode 100644 index e0fcc05c90a..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(zookeeper-server-3.8.0-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.8.0/pom.xml b/zookeeper-server/zookeeper-server-3.8.0/pom.xml deleted file mode 100644 index f6c8952849c..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/pom.xml +++ /dev/null @@ -1,123 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-parent</artifactId> - <version>8-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>zookeeper-server-3.8.0</artifactId> - <packaging>container-plugin</packaging> - <version>8-SNAPSHOT</version> - <properties> - <zookeeper.version>3.8.0</zookeeper.version> - </properties> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-client-common</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <!-- Don't use ZK version from zookeeper-client-common --> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>${zookeeper.version}</version> - <exclusions> - <!-- - Container provides wiring for all common log libraries - Duplicate embedding results in various warnings being printed to stderr - --> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- snappy-java and metrics-core are included here - to be able to work with ZooKeeper 3.7.0 due to - class loading issues --> - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.xerial.snappy</groupId> - <artifactId>snappy-java</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <forkMode>once</forkMode> - <systemPropertyVariables> - <zk-version>${zookeeper.version}</zk-version> - </systemPropertyVariables> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <importPackage>com.sun.management</importPackage> - <bundleSymbolicName>zookeeper-server</bundleSymbolicName> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java deleted file mode 100644 index e94110af2fb..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Starts or reconfigures zookeeper cluster. - * The QuorumPeer conditionally created here is owned by the Reconfigurer; - * when it already has a peer, that peer is used here in case start or shutdown is required. - * - * @author hmusum - */ -public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - - private QuorumPeer peer; - - @Inject - public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return true; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java deleted file mode 100644 index 66742b0e05b..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.ServerCnxn; -import org.apache.zookeeper.server.auth.AuthenticationProvider; -import org.apache.zookeeper.server.auth.X509AuthenticationProvider; - -import java.security.cert.X509Certificate; -import java.util.logging.Logger; - -/** - * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS - * - * @author bjorncs - */ -public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { - - private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); - - public VespaMtlsAuthenticationProvider() throws X509Exception { super(null, null);} - - @Override - public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { - // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. - // The client is authorized once the SSL handshake has completed. - X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); - if (certificateChain == null || certificateChain.length == 0) { - log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); - return KeeperException.Code.AUTHFAILED; - } - X509Certificate certificate = certificateChain[0]; - cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); - return KeeperException.Code.OK; - } - - @Override public String getScheme() { return "x509"; } - -} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java deleted file mode 100644 index 47ec03367c1..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.protect.Process; -import org.apache.zookeeper.server.admin.AdminServer; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; - -import java.io.IOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps - * exceptions so it can be used by code that does not depend on ZooKeeper. - * - * @author hmusum - */ -class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); - - @Override - public void start(Path path) { - initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); - } - - @Override - public void shutdown(Duration timeout) { - if (quorumPeer != null) { - log.log(Level.FINE, "Shutting down ZooKeeper server"); - try { - quorumPeer.shutdown(); - quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete - if (quorumPeer.isAlive()) - throw new IllegalStateException("Peer still alive after " + timeout); - } catch (RuntimeException | InterruptedException e) { - // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. - // - // When a VespaZooKeeperServer component receives a new config, the container will try to start a new - // server with the new config, this will fail until the old server is deconstructed. If the old server - // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is - // the better option. - Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); - } - } - } - - @Override - protected void initializeAndRun(String[] args) { - try { - super.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { - throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java deleted file mode 100644 index ae7bf8d84f5..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.admin.ZooKeeperAdmin; -import org.apache.zookeeper.data.ACL; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -@SuppressWarnings("unused") // Created by injection -public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); - - @Override - public void reconfigure(String connectionSpec, String servers) throws ReconfigException { - try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { - long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). - log.log(Level.INFO, "Applying ZooKeeper config: " + servers); - byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); - log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - - // Verify by issuing a write operation; this is only accepted once new quorum is obtained. - List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); - zooKeeperAdmin.delete(node, -1); - - log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } - catch ( KeeperException.ReconfigInProgress - | KeeperException.ConnectionLossException - | KeeperException.NewConfigNoQuorum e) { - throw new ReconfigException(e); - } - catch (KeeperException | IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException { - return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), - (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); - } - -} - diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 48f95d28910..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; -import java.time.Duration; - -/** - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaQuorumPeer peer; - private final ZooKeeperRunner runner; - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this.peer = new VespaQuorumPeer(); - this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); - } - - @Override - public void deconstruct() { - runner.shutdown(); - super.deconstruct(); - } - - @Override - public void shutdown() { - peer.shutdown(Duration.ofMinutes(1)); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return false; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/common/NetUtils.java deleted file mode 100644 index 33ec9b1303a..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.common; - -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.InetSocketAddress; - -/** - * This class contains common utilities for netstuff. Like printing IPv6 literals correctly - */ -public class NetUtils { - - // Note: Changed from original to use hostname from InetSocketAddress if there exists one - public static String formatInetAddr(InetSocketAddress addr) { - String hostName = addr.getHostName(); - if (hostName != null) { - return String.format("%s:%s", hostName, addr.getPort()); - } - - InetAddress ia = addr.getAddress(); - - if (ia == null) { - return String.format("%s:%s", addr.getHostString(), addr.getPort()); - } - if (ia instanceof Inet6Address) { - return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort()); - } else { - return String.format("%s:%s", ia.getHostAddress(), addr.getPort()); - } - } - - /** - * Separates host and port from given host port string if host port string is enclosed - * within square bracket. - * - * @param hostPort host port string - * @return String[]{host, port} if host port string is host:port - * or String[] {host, port:port} if host port string is host:port:port - * or String[] {host} if host port string is host - * or String[]{} if not a ipv6 host port string. - */ - public static String[] getIPV6HostAndPort(String hostPort) { - if (hostPort.startsWith("[")) { - int i = hostPort.lastIndexOf(']'); - if (i < 0) { - throw new IllegalArgumentException( - hostPort + " starts with '[' but has no matching ']'"); - } - String host = hostPort.substring(1, i); - if (host.isEmpty()) { - throw new IllegalArgumentException(host + " is empty."); - } - if (hostPort.length() > i + 1) { - return getHostPort(hostPort, i, host); - } - return new String[] { host }; - } else { - //Not an IPV6 host port string - return new String[] {}; - } - } - - private static String[] getHostPort(String hostPort, int indexOfClosingBracket, String host) { - // [127::1]:2181 , check separator : exits - if (hostPort.charAt(indexOfClosingBracket + 1) != ':') { - throw new IllegalArgumentException(hostPort + " does not have : after ]"); - } - // [127::1]: scenario - if (indexOfClosingBracket + 2 == hostPort.length()) { - throw new IllegalArgumentException(hostPort + " doesn't have a port after colon."); - } - //do not include - String port = hostPort.substring(indexOfClosingBracket + 2); - return new String[] { host, port }; - } -} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java deleted file mode 100644 index fdfe0fe8467..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.apache.zookeeper.server; - -import com.yahoo.vespa.zookeeper.Configurator; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.logging.Logger; - -/** - * Overrides secure setting with value from {@link Configurator}. - * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 - * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. - * - * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. - * - * @author bjorncs - */ -public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { - - private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); - - private final boolean isSecure; - - public VespaNettyServerCnxnFactory() { - super(); - this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; - boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); - log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); - } - - @Override - public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { - log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); - super.configure(addr, maxClientCnxns, backlog, isSecure); - } -} diff --git a/zookeeper-server/zookeeper-server-3.8.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.8.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java deleted file mode 100644 index db643d76e0d..00000000000 --- a/zookeeper-server/zookeeper-server-3.8.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.net.HostName; -import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; -import com.yahoo.vespa.zookeeper.Reconfigurer; -import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.admin.ZooKeeperAdmin; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.ServerSocket; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.IntStream; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static org.junit.Assert.assertEquals; - -public class VespaZooKeeperTest { - - static final Path tempDirRoot = getTmpDir(); - static final List<Integer> ports = new ArrayList<>(); - - /** - * Performs dynamic reconfiguration of ZooKeeper servers. - * - * First, a cluster of 3 servers is set up, and some data is written to it. - * Then, 3 new servers are added, and the first 3 marked for retirement; - * this should force the quorum to move the 3 new servers, but not disconnect the old ones. - * Next, the old servers are removed. - * Then, the cluster is reduced to size 1. - * Finally, the cluster grows to size 3 again. - * - * Throughout all of this, quorum should remain, and the data should remain the same. - */ - @Test(timeout = 120_000) - @Ignore // Unstable, some ZK server keeps resetting connections sometimes. - public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { - List<ZooKeeper> keepers = new ArrayList<>(); - for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); - for (int i = 0; i < 8; i++) keepers.get(i).run(); - - // Start the first three servers. - List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); - for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); - for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Wait for all servers to be up and running. - for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Write data to verify later. - String path = writeData(configs.get(0)); - - // Let three new servers join, causing the three older ones to retire and leave the ensemble. - configs = getConfigs(0, 3, 3, 3); - for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); - // The existing servers can't reconfigure and leave before the joiners are up. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Wait for new quorum to be established. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Verify written data is preserved. - verifyData(path, configs.get(3)); - - // Old servers are removed. - configs = getConfigs(3, 0, 3, 0); - for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); - // Old servers shut down, while the newer servers remain. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // Ensure old servers shut down properly. - for (int i = 0; i < 3; i++) keepers.get(i).await(); - // Ensure new servers have reconfigured. - for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Verify written data is preserved. - verifyData(path, configs.get(3)); - - - // Cluster shrinks to a single server. - configs = getConfigs(5, 0, 1, 0); - for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); - for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // We let the remaining server reconfigure the others out before they die. - for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 3; i < 5; i++) keepers.get(i).await(); - verifyData(path, configs.get(5)); - - // Cluster grows to 3 servers again. - configs = getConfigs(5, 0, 3, 2); - for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // Wait for the joiners. - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - verifyData(path, configs.get(7)); - - // Let the remaining servers terminate. - for (int i = 5; i < 8; i++) keepers.get(i).config = null; - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 5; i < 8; i++) keepers.get(i).await(); - } - - static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { - try (ZooKeeperAdmin admin = createAdmin(config)) { - List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); - String read = new String(admin.getData(node, false, new Stat()), UTF_8); - assertEquals("hi", read); - return node; - } - } - - static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { - for (int i = 0; i < 10; i++) { - try (ZooKeeperAdmin admin = createAdmin(config)) { - assertEquals("hi", new String(admin.getData(path, false, new Stat()), UTF_8)); - return; - } - catch (KeeperException.ConnectionLossException e) { - e.printStackTrace(); - Thread.sleep(10 << i); - } - } - } - - static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { - return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), - 10_000, - System.err::println, - new ZkClientConfigBuilder().toConfig()); - } - - static class ZooKeeper { - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - final Phaser phaser = new Phaser(2); - final AtomicReference<Future<?>> future = new AtomicReference<>(); - ZookeeperServerConfig config; - - void run() { - future.set(executor.submit(() -> { - Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); - phaser.arriveAndAwaitAdvance(); - while (config != null) { - new ReconfigurableVespaZooKeeperServer(reconfigurer, config); - phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here - phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff - } - reconfigurer.deconstruct(); - })); - } - - void await() throws ExecutionException, InterruptedException, TimeoutException { - future.get().get(30, SECONDS); - } - } - - static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { - return IntStream.rangeClosed(1, removed + retired + active) - .mapToObj(id -> getConfig(removed, retired, active, joining, id)) - .collect(toList()); - } - - // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. - static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { - if (id <= removed) - return null; - - Path tempDir = tempDirRoot.resolve("zookeeper-" + id); - return new ZookeeperServerConfig.Builder() - .clientPort(getPorts(id).get(0)) - .dataDir(tempDir.toString()) - .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) - .myid(id) - .myidFile(tempDir.resolve("myid").toString()) - .dynamicReconfiguration(true) - .server(IntStream.rangeClosed(removed + 1, removed + retired + active) - .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() - .id(i) - .clientPort(getPorts(i).get(0)) - .electionPort(getPorts(i).get(1)) - .quorumPort(getPorts(i).get(2)) - .hostname("localhost") - .joining(i - removed > retired + active - joining) - .retired(i - removed <= retired)) - .collect(toList())) - .build(); - } - - static List<Integer> getPorts(int id) { - if (ports.size() < id * 3) { - int previousPort; - if (ports.isEmpty()) { - String[] version = System.getProperty("zk-version").split("\\."); - int versionPortOffset = 0; - for (String part : version) - versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); - previousPort = 20000 + versionPortOffset % 30000; - } - else - previousPort = ports.get(ports.size() - 1); - - for (int i = 0; i < 3; i++) - ports.add(previousPort = nextPort(previousPort)); - } - return ports.subList(id * 3 - 3, id * 3); - } - - static int nextPort(int previousPort) { - for (int j = 1; j <= 30000; j++) { - int port = (previousPort + j); - while (port > 50000) - port -= 30000; - - try (ServerSocket socket = new ServerSocket(port)) { - return socket.getLocalPort(); - } - catch (IOException e) { - System.err.println("Could not bind port " + port + ": " + e); - } - } - throw new RuntimeException("No free ports"); - } - - static Path getTmpDir() { - try { - Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); - tempDir.toFile().deleteOnExit(); - return tempDir.toAbsolutePath(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index 8f8058c6c0b..6508c154978 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; @@ -47,7 +48,16 @@ public class Configurator { System.setProperty("zookeeper.snapshot.compression.method", zookeeperServerConfig.snapshotMethod()); } - void writeConfigToDisk() { writeConfigToDisk(VespaTlsConfig.fromSystem()); } + void writeConfigToDisk() { + VespaTlsConfig config; + String cfgFile = zookeeperServerConfig.vespaTlsConfigFile(); + if (cfgFile.isBlank()) { + config = VespaTlsConfig.fromSystem(); + } else { + config = VespaTlsConfig.fromConfig(Paths.get(cfgFile)); + } + writeConfigToDisk(config); + } // override of Vespa TLS config for unit testing void writeConfigToDisk(VespaTlsConfig vespaTlsConfig) { @@ -158,6 +168,7 @@ public class Configurator { default void appendSharedTlsConfig(StringBuilder builder, VespaTlsConfig vespaTlsConfig) { vespaTlsConfig.context().ifPresent(ctx -> { + VespaSslContextProvider.set(ctx); builder.append(configFieldPrefix()).append(".context.supplier.class=").append(VespaSslContextProvider.class.getName()).append("\n"); String enabledCiphers = Arrays.stream(ctx.parameters().getCipherSuites()).sorted().collect(Collectors.joining(",")); builder.append(configFieldPrefix()).append(".ciphersuites=").append(enabledCiphers).append("\n"); @@ -224,6 +235,13 @@ public class Configurator { TransportSecurityUtils.getInsecureMixedMode()); } + static VespaTlsConfig fromConfig(Path file) { + return new VespaTlsConfig( + new ConfigFileBasedTlsContext(file, TransportSecurityUtils.getInsecureAuthorizationMode()), + TransportSecurityUtils.getInsecureMixedMode()); + } + + static VespaTlsConfig tlsDisabled() { return new VespaTlsConfig(null, MixedMode.defaultValue()); } boolean tlsEnabled() { return context != null; } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaSslContextProvider.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaSslContextProvider.java index 89a0fa8a924..5434804cd62 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaSslContextProvider.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaSslContextProvider.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.security.tls.TlsContext; -import com.yahoo.security.tls.TransportSecurityUtils; import javax.net.ssl.SSLContext; import java.util.function.Supplier; @@ -14,12 +13,19 @@ import java.util.function.Supplier; */ public class VespaSslContextProvider implements Supplier<SSLContext> { - private static final SSLContext sslContext = TransportSecurityUtils.getSystemTlsContext().map(TlsContext::context).orElse(null); + private static TlsContext tlsContext; @Override public SSLContext get() { - if (sslContext == null) throw new IllegalStateException("Vespa TLS is not enabled"); - return sslContext; + synchronized (VespaSslContextProvider.class) { + if (tlsContext == null) throw new IllegalStateException("Vespa TLS is not enabled"); + return tlsContext.context(); + } + } + + static synchronized void set(TlsContext ctx) { + if (tlsContext != null) tlsContext.close(); + tlsContext = ctx; } } diff --git a/zookeeper-server/zookeeper-server/CMakeLists.txt b/zookeeper-server/zookeeper-server/CMakeLists.txt index c99916fafad..df7b71dfb42 100644 --- a/zookeeper-server/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(zookeeper-server-3.7.1-jar-with-dependencies.jar) +install_jar(zookeeper-server-3.8.0-jar-with-dependencies.jar) # Make symlink so that we have a default version, should be done only in zookeeper-server module -install_symlink(lib/jars/zookeeper-server-3.7.1-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) +install_symlink(lib/jars/zookeeper-server-3.8.0-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server/pom.xml b/zookeeper-server/zookeeper-server/pom.xml index ea0e61075f1..f6c8952849c 100644 --- a/zookeeper-server/zookeeper-server/pom.xml +++ b/zookeeper-server/zookeeper-server/pom.xml @@ -8,11 +8,11 @@ <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>zookeeper-server-3.7.1</artifactId> + <artifactId>zookeeper-server-3.8.0</artifactId> <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <properties> - <zookeeper.version>3.7.1</zookeeper.version> + <zookeeper.version>3.8.0</zookeeper.version> </properties> <dependencies> <dependency> |