diff options
423 files changed, 8638 insertions, 5726 deletions
diff --git a/client/go/Makefile b/client/go/Makefile index ce182b5d653..3da7b504f6c 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -40,7 +40,7 @@ all: test checkfmt install # $ git checkout vX.Y.Z # $ make dist-homebrew dist-homebrew: dist-version - brew bump-formula-pr --version $(VERSION) vespa-cli + brew bump-formula-pr --version $(VERSION) --no-browse vespa-cli # Create a GitHub release draft for all platforms. Note that this only creates a # draft, which is not publicly visible until it's explicitly published. diff --git a/client/go/auth/auth0/auth0.go b/client/go/auth/auth0/auth0.go index 52ba3f085a4..9b24066fa04 100644 --- a/client/go/auth/auth0/auth0.go +++ b/client/go/auth/auth0/auth0.go @@ -18,9 +18,7 @@ import ( "time" "github.com/lestrrat-go/jwx/jwt" - "github.com/pkg/browser" "github.com/vespa-engine/vespa/client/go/auth" - "github.com/vespa-engine/vespa/client/go/util" ) const accessTokenExpThreshold = 5 * time.Minute @@ -138,9 +136,7 @@ func (a *Auth0) IsLoggedIn() bool { } // PrepareSystem loads the System, refreshing its token if necessary. -// The System access token needs a refresh if: -// 1. the System scopes are different from the currently required scopes - (auth0 changes). -// 2. the access token is expired. +// The System access token needs a refresh if the access token has expired. func (a *Auth0) PrepareSystem(ctx context.Context) (*System, error) { if err := a.init(); err != nil { return nil, err @@ -150,11 +146,10 @@ func (a *Auth0) PrepareSystem(ctx context.Context) (*System, error) { return nil, err } - if s.AccessToken == "" || scopesChanged(s) { - s, err = RunLogin(ctx, a, true) - if err != nil { - return nil, err - } + if s.AccessToken == "" { + return nil, fmt.Errorf("access token missing: re-authenticate with 'vespa auth login'") + } else if scopesChanged(s) { + return nil, fmt.Errorf("authentication scopes cahnges: re-authenticate with 'vespa auth login'") } else if isExpired(s.ExpiresAt, accessTokenExpThreshold) { // check if the stored access token is expired: // use the refresh token to get a new access token: @@ -257,7 +252,7 @@ func (a *Auth0) AddSystem(s *System) error { return nil } -func (a *Auth0) removeSystem(s string) error { +func (a *Auth0) RemoveSystem(s string) error { _ = a.init() // If we're dealing with an empty file, we'll need to initialize this map. @@ -350,84 +345,3 @@ func (a *Auth0) initContext() (err error) { a.config = *cfg return nil } - -// RunLogin runs the login flow guiding the user through the process -// by showing the login instructions, opening the browser. -// Use `expired` to run the login from other commands setup: -// this will only affect the messages. -func RunLogin(ctx context.Context, a *Auth0, expired bool) (*System, error) { - if expired { - fmt.Println("Please sign in to re-authorize the CLI.") - } - - state, err := a.Authenticator.Start(ctx) - if err != nil { - return nil, fmt.Errorf("could not start the authentication process: %w", err) - } - - fmt.Printf("Your Device Confirmation code is: %s\n\n", state.UserCode) - - fmt.Println("If you prefer, you can open the URL directly for verification") - fmt.Printf("Your Verification URL: %s\n\n", state.VerificationURI) - - fmt.Println("Press Enter to open the browser to log in or ^C to quit...") - fmt.Scanln() - - err = browser.OpenURL(state.VerificationURI) - - if err != nil { - fmt.Printf("Couldn't open the URL, please do it manually: %s.", state.VerificationURI) - } - - var res auth.Result - err = util.Spinner(os.Stderr, "Waiting for login to complete in browser ...", func() error { - res, err = a.Authenticator.Wait(ctx, state) - return err - }) - - if err != nil { - return nil, fmt.Errorf("login error: %w", err) - } - - fmt.Print("\n") - fmt.Println("Successfully logged in.") - fmt.Print("\n") - - // store the refresh token - secretsStore := &auth.Keyring{} - err = secretsStore.Set(auth.SecretsNamespace, a.system, res.RefreshToken) - if err != nil { - // log the error but move on - fmt.Println("Could not store the refresh token locally, please expect to login again once your access token expired.") - } - - s := System{ - Name: a.system, - AccessToken: res.AccessToken, - ExpiresAt: time.Now().Add(time.Duration(res.ExpiresIn) * time.Second), - Scopes: auth.RequiredScopes(), - } - err = a.AddSystem(&s) - if err != nil { - return nil, fmt.Errorf("could not add system to config: %w", err) - } - - return &s, nil -} - -func RunLogout(a *Auth0) error { - s, err := a.getSystem() - if err != nil { - return err - } - - if err := a.removeSystem(s.Name); err != nil { - return err - } - - fmt.Print("\n") - fmt.Println("Successfully logged out.") - fmt.Print("\n") - - return nil -} diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index 5e23cdc9ebd..cacfe3fd180 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -28,13 +28,13 @@ key as necessary. It's possible to override the API key used through environment variables. This can be useful in continuous integration systems. -* VESPA_CLI_API_KEY containing the key directly: +Example of setting the key in-line: - export VESPA_CLI_API_KEY="my api key" + export VESPA_CLI_API_KEY="my api key" -* VESPA_CLI_API_KEY_FILE containing path to the key: +Example of loading the key from a custom path: - export VESPA_CLI_API_KEY_FILE=/path/to/api-key + export VESPA_CLI_API_KEY_FILE=/path/to/api-key Note that when overriding API key through environment variables, that key will always be used. It's not possible to specify a tenant-specific key.`, @@ -106,7 +106,7 @@ func printPublicKey(system vespa.System, apiKeyFile, tenant string) error { log.Printf("\nThis is your public key:\n%s", color.GreenString(string(pemPublicKey))) log.Printf("Its fingerprint is:\n%s\n", color.CyanString(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") - log.Printf(color.CyanString("%s/tenant/%s/keys"), system.ConsoleURL, tenant) + log.Printf(color.CyanString("%s/tenant/%s/account/keys"), system.ConsoleURL, tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") return nil } diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index caa32b0a963..11b20603ce8 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -36,13 +36,13 @@ environment variables. This can be useful in continuous integration systems. Example of setting the certificate and key in-line: -- export VESPA_CLI_DATA_PLANE_CERT="my cert" -- export VESPA_CLI_DATA_PLANE_KEY="my private key" + export VESPA_CLI_DATA_PLANE_CERT="my cert" + export VESPA_CLI_DATA_PLANE_KEY="my private key" Example of loading certificate and key from custom paths: -- export VESPA_CLI_DATA_PLANE_CERT_FILE=/path/to/cert -- export VESPA_CLI_DATA_PLANE_KEY_FILE=/path/to/key + export VESPA_CLI_DATA_PLANE_CERT_FILE=/path/to/cert + export VESPA_CLI_DATA_PLANE_KEY_FILE=/path/to/key Note that when overriding key pair through environment variables, that key pair will always be used for all applications. It's not possible to specify an diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index 180ec18debf..0829e064af9 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -19,7 +19,6 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/util" ) func newCloneCmd(cli *CLI) *cobra.Command { @@ -116,7 +115,7 @@ func fetchSampleAppsZip(destination string, cli *CLI) error { return fmt.Errorf("could not create temporary file: %w", err) } defer f.Close() - return util.Spinner(cli.Stderr, color.YellowString("Downloading sample apps ..."), func() error { + return cli.spinner(cli.Stderr, color.YellowString("Downloading sample apps ..."), func() error { request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil) if err != nil { return fmt.Errorf("invalid url: %w", err) diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index cf2176b0c69..fbecd4e19de 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -12,7 +12,6 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -54,7 +53,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, opts := cli.createDeploymentOptions(pkg, target) var result vespa.PrepareResult - err = util.Spinner(cli.Stderr, "Uploading application package ...", func() error { + err = cli.spinner(cli.Stderr, "Uploading application package ...", func() error { result, err = vespa.Deploy(opts) return err }) @@ -103,7 +102,7 @@ func newPrepareCmd(cli *CLI) *cobra.Command { } opts := cli.createDeploymentOptions(pkg, target) var result vespa.PrepareResult - err = util.Spinner(cli.Stderr, "Uploading application package ...", func() error { + err = cli.spinner(cli.Stderr, "Uploading application package ...", func() error { result, err = vespa.Prepare(opts) return err }) diff --git a/client/go/cmd/log_test.go b/client/go/cmd/log_test.go index 4a6bd89c12d..fb08474dd4d 100644 --- a/client/go/cmd/log_test.go +++ b/client/go/cmd/log_test.go @@ -46,6 +46,6 @@ func TestLogOldClient(t *testing.T) { assert.Nil(t, cli.Run("auth", "cert", pkgDir)) assert.Nil(t, cli.Run("log")) - expected := "Error: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This is not a fatal error, but this version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" + expected := "Warning: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" assert.Contains(t, stderr.String(), expected) } diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go index 3750037be88..5cf471ed8db 100644 --- a/client/go/cmd/login.go +++ b/client/go/cmd/login.go @@ -1,10 +1,21 @@ package cmd import ( + "fmt" + "log" + "os" + "time" + + "github.com/pkg/browser" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/auth" "github.com/vespa-engine/vespa/client/go/auth/auth0" ) +// newLoginCmd runs the login flow guiding the user through the process +// by showing the login instructions, opening the browser. +// Use `expired` to run the login from other commands setup: +// this will only affect the messages. func newLoginCmd(cli *CLI) *cobra.Command { return &cobra.Command{ Use: "login", @@ -27,7 +38,57 @@ func newLoginCmd(cli *CLI) *cobra.Command { if err != nil { return err } - _, err = auth0.RunLogin(ctx, a, false) + state, err := a.Authenticator.Start(ctx) + if err != nil { + return fmt.Errorf("could not start the authentication process: %w", err) + } + + log.Printf("Your Device Confirmation code is: %s\n\n", state.UserCode) + + log.Println("If you prefer, you can open the URL directly for verification") + log.Printf("Your Verification URL: %s\n\n", state.VerificationURI) + + log.Println("Press Enter to open the browser to log in or ^C to quit...") + fmt.Scanln() + + err = browser.OpenURL(state.VerificationURI) + + if err != nil { + log.Printf("Couldn't open the URL, please do it manually: %s.", state.VerificationURI) + } + + var res auth.Result + err = cli.spinner(os.Stderr, "Waiting for login to complete in browser ...", func() error { + res, err = a.Authenticator.Wait(ctx, state) + return err + }) + + if err != nil { + return fmt.Errorf("login error: %w", err) + } + + log.Print("\n") + log.Println("Successfully logged in.") + log.Print("\n") + + // store the refresh token + secretsStore := &auth.Keyring{} + err = secretsStore.Set(auth.SecretsNamespace, system.Name, res.RefreshToken) + if err != nil { + // log the error but move on + log.Println("Could not store the refresh token locally, please expect to login again once your access token expired.") + } + + s := auth0.System{ + Name: system.Name, + AccessToken: res.AccessToken, + ExpiresAt: time.Now().Add(time.Duration(res.ExpiresIn) * time.Second), + Scopes: auth.RequiredScopes(), + } + err = a.AddSystem(&s) + if err != nil { + return fmt.Errorf("could not add system to config: %w", err) + } return err }, } diff --git a/client/go/cmd/logout.go b/client/go/cmd/logout.go index 6cef5ee371c..c0a6b60ab2e 100644 --- a/client/go/cmd/logout.go +++ b/client/go/cmd/logout.go @@ -1,6 +1,8 @@ package cmd import ( + "log" + "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/auth/auth0" ) @@ -26,8 +28,14 @@ func newLogoutCmd(cli *CLI) *cobra.Command { if err != nil { return err } - err = auth0.RunLogout(a) - return err + if err := a.RemoveSystem(system.Name); err != nil { + return err + } + + log.Print("\n") + log.Println("Successfully logged out.") + log.Print("\n") + return nil }, } } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 2dbc0e89e0f..49cab75be53 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -52,6 +52,7 @@ type CLI struct { httpClient util.HTTPClient exec executor isTerminal func() bool + spinner func(w io.Writer, message string, fn func() error) error } // Flags holds the global Flags of Vespa CLI. @@ -136,11 +137,12 @@ Vespa documentation: https://docs.vespa.ai`, httpClient: util.CreateClient(time.Second * 10), exec: &execSubprocess{}, } - cli.isTerminal = func() bool { return isTerminal(cli.Stdout) || isTerminal(cli.Stderr) } + cli.isTerminal = func() bool { return isTerminal(cli.Stdout) && isTerminal(cli.Stderr) } cli.configureFlags() if err := cli.loadConfig(); err != nil { return nil, err } + cli.configureSpinner() cli.configureCommands() cmd.PersistentPreRunE = cli.configureOutput return &cli, nil @@ -203,6 +205,19 @@ func (c *CLI) configureFlags() { c.flags = &flags } +func (c *CLI) configureSpinner() { + // Explicitly disable spinner for Screwdriver. It emulates a tty but + // \r result in a newline, and output gets truncated. + _, screwdriver := c.Environment["SCREWDRIVER"] + if c.flags.quiet || !c.isTerminal() || screwdriver { + c.spinner = func(w io.Writer, message string, fn func() error) error { + return fn() + } + } else { + c.spinner = util.Spinner + } +} + func (c *CLI) configureCommands() { rootCmd := c.cmd authCmd := newAuthCmd() @@ -272,7 +287,7 @@ func (c *CLI) target(opts targetOptions) (vespa.Target, error) { } if !c.isCloudCI() { // Vespa Cloud always runs an up-to-date version if err := target.CheckVersion(c.version); err != nil { - c.printErrHint(err, "This is not a fatal error, but this version may not work as expected", "Try 'vespa version' to check for a new version") + c.printWarning(err.Error(), "This version may not work as expected", "Try 'vespa version' to check for a new version") } } return target, nil diff --git a/client/go/util/spinner.go b/client/go/util/spinner.go index bdce2dbfabe..39b00352c32 100644 --- a/client/go/util/spinner.go +++ b/client/go/util/spinner.go @@ -4,12 +4,10 @@ package util import ( "io" - "os" "strings" "time" "github.com/briandowns/spinner" - "github.com/mattn/go-isatty" ) // Spinner writes message to writer w and executes function fn. While fn is running a spinning animation will be @@ -24,21 +22,11 @@ func Spinner(w io.Writer, message string, fn func() error) error { } s.Prefix = message s.FinalMSG = "\r" + message + "done\n" - isTerminal := isTerminal(w) - if isTerminal { // spinner package does this check too, but it's hardcoded to check os.Stdout :( - s.Start() - } + s.Start() err := fn() - if isTerminal { - s.Stop() - } if err != nil { s.FinalMSG = "\r" + message + "failed\n" } + s.Stop() return err } - -func isTerminal(w io.Writer) bool { - f, ok := w.(*os.File) - return ok && isatty.IsTerminal(f.Fd()) -} diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index 1eca2d18493..5f606c68c1e 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -17,6 +17,8 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; /** @@ -30,6 +32,8 @@ public class ClusterController extends AbstractComponent private final JDiscMetricWrapper metricWrapper; private final Map<String, FleetController> controllers = new TreeMap<>(); private final Map<String, StatusHandler.ContainerStatusPageServer> status = new TreeMap<>(); + private final AtomicInteger referents = new AtomicInteger(); + private final AtomicBoolean shutdown = new AtomicBoolean(); /** * Dependency injection constructor for controller. A {@link VespaZooKeeperServer} argument is required @@ -43,6 +47,7 @@ public class ClusterController extends AbstractComponent } public void setOptions(FleetControllerOptions options, Metric metricImpl) throws Exception { + referents.incrementAndGet(); metricWrapper.updateMetricImplementation(metricImpl); verifyThatZooKeeperWorks(options); synchronized (controllers) { @@ -60,16 +65,32 @@ public class ClusterController extends AbstractComponent @Override public void deconstruct() { - synchronized (controllers) { - for (FleetController controller : controllers.values()) { - try{ - shutdownController(controller); - } catch (Exception e) { - log.warning("Failed to shut down fleet controller: " + e.getMessage()); + shutdown(); + } + + /** + * Since we hack around injecting a running ZK here by providing one through the configurer instead, + * we must also let the last configurer shut down this controller, to ensure this is shut down + * before the ZK server it had injected from the configurers. + */ + void countdown() { + if (referents.decrementAndGet() == 0) + shutdown(); + } + + void shutdown() { + if (shutdown.compareAndSet(false, true)) { + synchronized (controllers) { + for (FleetController controller : controllers.values()) { + try { + shutdownController(controller); + } + catch (Exception e) { + log.warning("Failed to shut down fleet controller: " + e.getMessage()); + } } } } - super.deconstruct(); } @Override @@ -79,6 +100,8 @@ public class ClusterController extends AbstractComponent } } + FleetController getController(String name) { return controllers.get(name); } + @Override public StatusHandler.ContainerStatusPageServer get(String cluster) { return status.get(cluster); diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index 75e838bae3e..a0e290de172 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.NodeType; @@ -19,9 +20,10 @@ import java.util.Map; * When the cluster controller is reconfigured, a new instance of this is created, which will propagate configured * options to receivers such as the fleet controller. */ -public class ClusterControllerClusterConfigurer { +public class ClusterControllerClusterConfigurer extends AbstractComponent { private final FleetControllerOptions options; + private final ClusterController controller; /** * The {@link VespaZooKeeperServer} argument is required by the injected {@link ClusterController}, @@ -37,9 +39,13 @@ public class ClusterControllerClusterConfigurer { Metric metricImpl, VespaZooKeeperServer started) throws Exception { this.options = configure(distributionConfig, fleetcontrollerConfig, slobroksConfig, zookeepersConfig); - if (controller != null) { - controller.setOptions(options, metricImpl); - } + this.controller = controller; + if (controller != null) controller.setOptions(options, metricImpl); + } + + @Override + public void deconstruct() { + if (controller != null) controller.countdown(); } FleetControllerOptions getOptions() { return options; } diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java index f6e64dc7186..dda18fc3396 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -50,7 +51,7 @@ public class ClusterControllerClusterConfigurerTest { @Override public Context createContext(Map<String, ?> stringMap) { return null; } }; - // Used in standalone modus to get config without a cluster controller instance + // Used in standalone mode to get config without a cluster controller instance ClusterControllerClusterConfigurer configurer = new ClusterControllerClusterConfigurer( null, new StorDistributionConfig(distributionConfig), @@ -60,7 +61,7 @@ public class ClusterControllerClusterConfigurerTest { metric, null ); - assertTrue(configurer.getOptions() != null); + assertNotNull(configurer.getOptions()); assertEquals(0.123, configurer.getOptions().minNodeRatioPerGroup, 0.01); assertTrue(configurer.getOptions().clusterFeedBlockEnabled); assertEquals(0.5, configurer.getOptions().clusterFeedBlockLimit.get("foo"), 0.01); diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java index 5c9a7eb9c0c..2c33f781737 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerTest.java @@ -11,8 +11,11 @@ import org.junit.Test; import java.util.Map; import java.util.Set; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Doesn't really test cluster controller, but runs some lines of code. @@ -41,17 +44,16 @@ public class ClusterControllerTest { @Test public void testSimple() throws Exception { - // Cluster controller object keeps state and should never be remade, so should - // inject nothing ClusterController cc = new ClusterController(); cc.setOptions(options, metric); cc.setOptions(options, metric); - cc.getFleetControllers(); - cc.getAll(); - + assertEquals(1, cc.getFleetControllers().size()); assertNotNull(cc.get("storage")); assertNull(cc.get("music")); - cc.deconstruct(); + cc.countdown(); + assertTrue(cc.getController("storage").isRunning()); + cc.countdown(); + assertFalse(cc.getController("storage").isRunning()); } @Test @@ -62,7 +64,7 @@ public class ClusterControllerTest { } }; cc.setOptions(options, metric); - cc.deconstruct(); + cc.countdown(); } } diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index 44d729c802e..94b65a01e57 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -14,6 +14,8 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.yolean.Exceptions; +import java.io.Closeable; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -29,7 +31,7 @@ import static java.util.stream.Collectors.toUnmodifiableMap; * * @author jonmv */ -public class ReindexingCurator { +public class ReindexingCurator implements Closeable { private static final Logger log = Logger.getLogger(ReindexingCurator.class.getName()); @@ -94,6 +96,11 @@ public class ReindexingCurator { private Path statusPath(String clusterName) { return rootPath(clusterName).append("status"); } private Path lockPath(String clusterName) { return rootPath(clusterName).append("lock"); } + @Override + public void close() { + curator.close(); + } + private static class ReindexingSerializer { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java index a9642c591ea..afb491fa293 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java @@ -103,9 +103,9 @@ public class ReindexingMaintainer extends AbstractComponent { executor.awaitTermination(5, TimeUnit.SECONDS); // Give it 5s to complete gracefully. - curator.close(); // Close the underlying curator independently to force shutdown + curator.close(); // Close the underlying curator independently to force shutdown. - if ( !executor.isShutdown() && ! executor.awaitTermination(5, TimeUnit.SECONDS)) + if ( ! executor.isShutdown() && ! executor.awaitTermination(5, TimeUnit.SECONDS)) log.log(WARNING, "Failed to shut down reindexing within timeout"); } catch (InterruptedException e) { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java index 8d9d21999d5..7816686221b 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java @@ -65,6 +65,11 @@ public class ReindexingV1ApiHandler extends ThreadedHttpRequestHandler { return ErrorResponse.notFoundError("Nothing at " + request.getUri().getRawPath()); } + @Override + public void destroy() { + database.close(); + } + HttpResponse getRoot() { Slime slime = new Slime(); slime.setObject().setArray("resources").addObject().setString("url", "/reindexing/v1/status"); @@ -88,7 +93,6 @@ public class ReindexingV1ApiHandler extends ThreadedHttpRequestHandler { return new SlimeJsonResponse(slime); } - private static String toString(Reindexing.State state) { switch (state) { case READY: return "pending"; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index a7fd48bfea8..54d22d58832 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -70,6 +70,9 @@ public interface Model { /** Returns the version of this model. */ default Version version() { return Version.emptyVersion; } + /** Returns the wanted node version of this model. */ + default Version wantedNodeVersion() { return Version.emptyVersion; } + /** Returns the provisioned hosts of this. */ default Provisioned provisioned() { return new Provisioned(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 27286a7dbbe..704921a3bb7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -78,7 +78,7 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } private boolean useV8GeoPositions = false; - private boolean useV8DocManagerCfg = false; + private boolean useV8DocManagerCfg = true; public void useFeatureFlags(ModelContext.FeatureFlags featureFlags) { this.useV8GeoPositions = featureFlags.useV8GeoPositions(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 480b6590555..2c028c52bf8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -77,6 +77,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean persistenceThrottlingOfMergeFeedOps = true; private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; + private boolean useV8DocManagerCfg = true; private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; private boolean experimentalSdParsing = false; @@ -137,6 +138,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; } @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } + @Override public boolean useV8DocManagerCfg() { return useV8DocManagerCfg; } @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } @Override public boolean experimentalSdParsing() { return this.experimentalSdParsing; } @@ -367,6 +369,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setUseV8DocManagerCfg(boolean value) { + this.useV8DocManagerCfg = value; + return this; + } + public TestProperties setEnvironmentVariables(List<String> value) { this.environmentVariables = value; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index 16eef798acd..2dda670f07c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -30,7 +30,6 @@ public class Application { private final ApplicationPackage applicationPackage; private final Map<String, Schema> schemas; private final DocumentModel documentModel; - private final RankProfileRegistry rankProfileRegistry; public Application(ApplicationPackage applicationPackage, List<Schema> schemas, @@ -50,7 +49,6 @@ public class Application { schemaMap.put(schema.getName(), schema); } this.schemas = Collections.unmodifiableMap(schemaMap); - this.rankProfileRegistry = rankProfileRegistry; schemas.forEach(schema -> schema.setOwner(this)); if (validate) @@ -84,7 +82,6 @@ public class Application { List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForStructs().processSchemaFields(schema); new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? new Processing(properties).process(schema, logger, @@ -103,8 +100,6 @@ public class Application { /** Returns an unmodifiable list of the schemas of this application */ public Map<String, Schema> schemas() { return schemas; } - public RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; } - public DocumentModel documentModel() { return documentModel; } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 88a98b94ddc..9b92f1aa80c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -32,8 +32,9 @@ import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -155,9 +156,9 @@ public class DocumentModelBuilder { private static void addSearchField(SDField field, SearchDef searchDef) { SearchField searchField = - new SearchField(field, - field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), - field.getAttributes().containsKey(field.getName())); + new SearchField(field, + field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), + field.getAttributes().containsKey(field.getName())); searchDef.add(searchField); // Add field to views @@ -189,11 +190,14 @@ public class DocumentModelBuilder { } // This is how you make a "Pair" class in java.... - private static class TypeReplacement extends AbstractMap.SimpleEntry<DataType,DataType> { - DataType oldType() { return getKey(); } - DataType newType() { return getValue(); } + private static class TypeReplacement { + private final DataType oldType; + private final DataType newType; + DataType oldType() { return oldType; } + DataType newType() { return newType; } public TypeReplacement(DataType oldType, DataType newType) { - super(oldType, newType); + this.oldType = oldType; + this.newType = newType; } } @@ -208,15 +212,15 @@ public class DocumentModelBuilder { lst.add(convert(doc)); model.getDocumentManager().add(lst.getLast()); } - Set<TypeReplacement> replacements = new HashSet<>(); + Map<DataType, DataType> replacements = new IdentityHashMap<>(); for(NewDocumentType doc : lst) { resolveTemporaries(doc.getAllTypes(), lst, replacements); } for(NewDocumentType doc : lst) { - for (var entry : replacements) { - var old = entry.oldType(); + for (var entry : replacements.entrySet()) { + var old = entry.getKey(); if (doc.getDataType(old.getId()) == old) { - doc.replace(entry.newType()); + doc.replace(entry.getValue()); } } } @@ -224,8 +228,7 @@ public class DocumentModelBuilder { private static void resolveTemporaries(DataTypeCollection dtc, Collection<NewDocumentType> docs, - Set<TypeReplacement> replacements) - { + Map<DataType, DataType> replacements) { for (DataType type : dtc.getTypes()) { resolveTemporariesRecurse(type, dtc, docs, replacements); } @@ -234,7 +237,10 @@ public class DocumentModelBuilder { @SuppressWarnings("deprecation") private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, Collection<NewDocumentType> docs, - Set<TypeReplacement> replacements) { + Map<DataType, DataType> replacements) { + if (replacements.containsKey(type)) { + return replacements.get(type); + } DataType original = type; if (type instanceof TemporaryStructuredDataType) { DataType other = repo.getDataType(type.getId()); @@ -244,7 +250,15 @@ public class DocumentModelBuilder { if (other != null) { type = other; } - } else if (type instanceof DocumentType || type instanceof NewDocumentType) { + } else if (type instanceof DocumentType) { + DataType other = getDocumentType(docs, type.getId()); + if (other != null) { + type = other; + } else if (! type.getName().equals("document")) { + throw new IllegalArgumentException + ("Can not handle nested document definitions. Undefined document type: " + type.toString()); + } + } else if (type instanceof NewDocumentType) { DataType other = getDocumentType(docs, type.getId()); if (other != null) { type = other; @@ -264,21 +278,31 @@ public class DocumentModelBuilder { } else if (type instanceof MapDataType) { MapDataType t = (MapDataType) type; - var kt = resolveTemporariesRecurse(t.getKeyType(), repo, docs, replacements); - var vt = resolveTemporariesRecurse(t.getValueType(), repo, docs, replacements); - type = new MapDataType(kt, vt, t.getId()); + var old_kt = t.getKeyType(); + var old_vt = t.getValueType(); + var kt = resolveTemporariesRecurse(old_kt, repo, docs, replacements); + var vt = resolveTemporariesRecurse(old_vt, repo, docs, replacements); + if (kt != old_kt || vt != old_vt) { + type = new MapDataType(kt, vt, t.getId()); + } } else if (type instanceof ArrayDataType) { ArrayDataType t = (ArrayDataType) type; - var nt = resolveTemporariesRecurse(t.getNestedType(), repo, docs, replacements); - type = new ArrayDataType(nt, t.getId()); + var old_nt = t.getNestedType(); + var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); + if (nt != old_nt) { + type = new ArrayDataType(nt, t.getId()); + } } else if (type instanceof WeightedSetDataType) { WeightedSetDataType t = (WeightedSetDataType) type; - var nt = resolveTemporariesRecurse(t.getNestedType(), repo, docs, replacements); - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - type = new WeightedSetDataType(nt, c, r, t.getId()); + var old_nt = t.getNestedType(); + var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); + if (nt != old_nt) { + boolean c = t.createIfNonExistent(); + boolean r = t.removeIfZero(); + type = new WeightedSetDataType(nt, c, r, t.getId()); + } } else if (type instanceof ReferenceDataType) { ReferenceDataType t = (ReferenceDataType) type; @@ -288,7 +312,7 @@ public class DocumentModelBuilder { } } if (type != original) { - replacements.add(new TypeReplacement(original, type)); + replacements.put(original, type); } return type; } @@ -302,89 +326,6 @@ public class DocumentModelBuilder { return null; } - @SuppressWarnings("deprecation") - private static void specialHandleAnnotationReference(NewDocumentType docType, Field field) { - DataType fieldType = specialHandleAnnotationReferenceRecurse(docType, field.getName(), field.getDataType()); - if (fieldType == null) { - return; - } - field.setDataType(fieldType); // XXX deprecated - } - - private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName, - DataType dataType) { - if (dataType instanceof TemporaryAnnotationReferenceDataType) { - TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; - if (refType.getId() != 0) { - return null; - } - AnnotationType target = docType.getAnnotationType(refType.getTarget()); - if (target == null) { - throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + - "' does not exist."); - } - dataType = new AnnotationReferenceDataType(target); - addType(docType, dataType); - return dataType; - } - else if (dataType instanceof MapDataType) { - MapDataType t = (MapDataType)dataType; - DataType valueType = specialHandleAnnotationReferenceRecurse(docType, fieldName, t.getValueType()); - if (valueType == null) { - return null; - } - var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); - addType(docType, mapType); - return mapType; - } - else if (dataType instanceof ArrayDataType) { - ArrayDataType t = (ArrayDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, t.getNestedType()); - if (nestedType == null) { - return null; - } - var lstType = new ArrayDataType(nestedType, t.getId()); - addType(docType, lstType); - return lstType; - } - else if (dataType instanceof WeightedSetDataType) { - WeightedSetDataType t = (WeightedSetDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, t.getNestedType()); - if (nestedType == null) { - return null; - } - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); - addType(docType, lstType); - return lstType; - } - return null; - } - - private static StructDataType handleStruct(NewDocumentType dt, SDDocumentType type) { - StructDataType s = new StructDataType(type.getName()); - for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { - specialHandleAnnotationReference(dt, f); - s.addField(f); - } - for (StructDataType inherited : type.getDocumentType().contentStruct().getInheritedTypes()) { - s.inherit(inherited); - } - extractNestedTypes(dt, s); - addType(dt, s); - return s; - } - - private static StructDataType handleStruct(NewDocumentType dt, StructDataType s) { - for (Field f : s.getFieldsThisTypeOnly()) { - specialHandleAnnotationReference(dt, f); - } - extractNestedTypes(dt, s); - addType(dt, s); - return s; - } - private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { if (sa.getInherits() != null) { AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); @@ -395,8 +336,6 @@ public class DocumentModelBuilder { } private NewDocumentType convert(SDDocumentType sdoc) { - Map<AnnotationType, String> annotationInheritance = new HashMap<>(); - Map<StructDataType, String> structInheritance = new HashMap<>(); NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), sdoc.getDocumentType().contentStruct(), sdoc.getFieldSets(), @@ -404,63 +343,231 @@ public class DocumentModelBuilder { convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); - NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); - if (inherited != null) { - dt.inherit(inherited); - } - } - for (SDDocumentType type : sdoc.getTypes()) { - if (type.isStruct()) { - handleStruct(dt, type); - } else { - throw new IllegalArgumentException("Data type '" + sdoc.getName() + "' is not a struct => tostring='" + sdoc.toString() + "'."); - } - } - for (SDDocumentType type : sdoc.getTypes()) { - for (SDDocumentType proxy : type.getInheritedTypes()) { - var inherited = dt.getDataTypeRecursive(proxy.getName()); - var converted = (StructDataType) dt.getDataType(type.getName()); - converted.inherit((StructDataType) inherited); + NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); + if (inherited != null) { + dt.inherit(inherited); } } - for (AnnotationType annotation : sdoc.getAnnotations().values()) { - dt.add(annotation); + var extractor = new TypeExtractor(dt); + extractor.extract(sdoc); + return dt; + } + + static class TypeExtractor { + private final NewDocumentType targetDt; + Map<AnnotationType, String> annotationInheritance = new LinkedHashMap<>(); + Map<StructDataType, String> structInheritance = new LinkedHashMap<>(); + private final Map<Object, Object> inProgress = new IdentityHashMap<>(); + TypeExtractor(NewDocumentType target) { + this.targetDt = target; } - for (AnnotationType annotation : sdoc.getAnnotations().values()) { - SDAnnotationType sa = (SDAnnotationType) annotation; - if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { - annotationInheritance.put(annotation, sa.getInherits()); + + void extract(SDDocumentType sdoc) { + for (SDDocumentType type : sdoc.getTypes()) { + if (type.isStruct()) { + handleStruct(type); + } else { + throw new IllegalArgumentException("Data type '" + type.getName() + "' is not a struct => tostring='" + type.toString() + "'."); + } } - if (annotation.getDataType() == null) { - if (sa.getSdDocType() != null) { - StructDataType s = handleStruct(dt, sa.getSdDocType()); - annotation.setDataType(s); - if ((sa.getInherits() != null)) { - structInheritance.put(s, "annotation."+sa.getInherits()); + for (SDDocumentType type : sdoc.getTypes()) { + for (SDDocumentType proxy : type.getInheritedTypes()) { + var inherited = (StructDataType) targetDt.getDataTypeRecursive(proxy.getName()); + var converted = (StructDataType) targetDt.getDataType(type.getName()); + if (! converted.inherits(inherited)) { + converted.inherit(inherited); } - } else if (sa.getInherits() != null) { - StructDataType s = new StructDataType("annotation."+annotation.getName()); - if (anyParentsHavePayLoad(sa, sdoc)) { + } + } + for (AnnotationType annotation : sdoc.getAnnotations().values()) { + targetDt.add(annotation); + } + for (AnnotationType annotation : sdoc.getAnnotations().values()) { + SDAnnotationType sa = (SDAnnotationType) annotation; + if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { + annotationInheritance.put(annotation, sa.getInherits()); + } + if (annotation.getDataType() == null) { + if (sa.getSdDocType() != null) { + StructDataType s = handleStruct(sa.getSdDocType()); annotation.setDataType(s); - addType(dt, s); + if ((sa.getInherits() != null)) { + structInheritance.put(s, "annotation."+sa.getInherits()); + } + } else if (sa.getInherits() != null) { + StructDataType s = new StructDataType("annotation."+annotation.getName()); + if (anyParentsHavePayLoad(sa, sdoc)) { + annotation.setDataType(s); + addType(s); + } + structInheritance.put(s, "annotation."+sa.getInherits()); } - structInheritance.put(s, "annotation."+sa.getInherits()); } } + for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) { + e.getKey().inherit(targetDt.getAnnotationType(e.getValue())); + } + for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) { + StructDataType s = (StructDataType)targetDt.getDataType(e.getValue()); + if (s != null) { + e.getKey().inherit(s); + } + } + handleStruct(sdoc.getDocumentType().contentStruct()); + extractDataTypesFromFields(sdoc.fieldSet()); } - for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) { - e.getKey().inherit(dt.getAnnotationType(e.getValue())); + + private void extractDataTypesFromFields(Collection<Field> fields) { + for (Field f : fields) { + DataType type = f.getDataType(); + if (testAddType(type)) { + extractNestedTypes(type); + addType(type); + } + } } - for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) { - StructDataType s = (StructDataType)dt.getDataType(e.getValue()); - if (s != null) { - e.getKey().inherit(s); + + private void extractNestedTypes(DataType type) { + if (inProgress.containsKey(type)) { + return; + } + inProgress.put(type, this); + if (type instanceof StructDataType) { + StructDataType tmp = (StructDataType) type; + extractDataTypesFromFields(tmp.getFieldsThisTypeOnly()); + } else if (type instanceof CollectionDataType) { + CollectionDataType tmp = (CollectionDataType) type; + extractNestedTypes(tmp.getNestedType()); + addType(tmp.getNestedType()); + } else if (type instanceof MapDataType) { + MapDataType tmp = (MapDataType) type; + extractNestedTypes(tmp.getKeyType()); + extractNestedTypes(tmp.getValueType()); + addType(tmp.getKeyType()); + addType(tmp.getValueType()); + } else if (type instanceof TemporaryAnnotationReferenceDataType) { + throw new IllegalArgumentException(type.toString()); } } - handleStruct(dt, sdoc.getDocumentType().contentStruct()); - extractDataTypesFromFields(dt, sdoc.fieldSet()); - return dt; + private boolean testAddType(DataType type) { return internalAddType(type, true); } + + private boolean addType(DataType type) { return internalAddType(type, false); } + + private boolean internalAddType(DataType type, boolean dryRun) { + DataType oldType = targetDt.getDataTypeRecursive(type.getId()); + if (oldType == null) { + if ( ! dryRun) { + targetDt.add(type); + } + return true; + } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { + StructDataType s = (StructDataType) type; + StructDataType os = (StructDataType) oldType; + if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { + if ( ! dryRun) { + targetDt.replace(type); + } + return true; + } + } + return false; + } + + + @SuppressWarnings("deprecation") + private void specialHandleAnnotationReference(Field field) { + DataType fieldType = specialHandleAnnotationReferenceRecurse(field.getName(), field.getDataType()); + if (fieldType == null) { + return; + } + field.setDataType(fieldType); // XXX deprecated + } + + private DataType specialHandleAnnotationReferenceRecurse(String fieldName, + DataType dataType) { + if (dataType instanceof TemporaryAnnotationReferenceDataType) { + TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; + if (refType.getId() != 0) { + return null; + } + AnnotationType target = targetDt.getAnnotationType(refType.getTarget()); + if (target == null) { + throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + + "' does not exist."); + } + dataType = new AnnotationReferenceDataType(target); + addType(dataType); + return dataType; + } + else if (dataType instanceof MapDataType) { + MapDataType t = (MapDataType)dataType; + DataType valueType = specialHandleAnnotationReferenceRecurse(fieldName, t.getValueType()); + if (valueType == null) { + return null; + } + var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); + addType(mapType); + return mapType; + } + else if (dataType instanceof ArrayDataType) { + ArrayDataType t = (ArrayDataType) dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + if (nestedType == null) { + return null; + } + var lstType = new ArrayDataType(nestedType, t.getId()); + addType(lstType); + return lstType; + } + else if (dataType instanceof WeightedSetDataType) { + WeightedSetDataType t = (WeightedSetDataType) dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + if (nestedType == null) { + return null; + } + boolean c = t.createIfNonExistent(); + boolean r = t.removeIfZero(); + var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); + addType(lstType); + return lstType; + } + return null; + } + + @SuppressWarnings("deprecation") + private StructDataType handleStruct(SDDocumentType type) { + if (type.isStruct()) { + var st = type.getStruct(); + if (st.getName().equals(type.getName()) && + (st instanceof StructDataType) && + ! (st instanceof TemporaryStructuredDataType)) + { + return handleStruct((StructDataType) st); + } + } + StructDataType s = new StructDataType(type.getName()); + for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(f); + s.addField(f); + } + for (StructDataType inherited : type.getDocumentType().contentStruct().getInheritedTypes()) { + s.inherit(inherited); + } + extractNestedTypes(s); + addType(s); + return s; + } + + private StructDataType handleStruct(StructDataType s) { + for (Field f : s.getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(f); + } + extractNestedTypes(s); + addType(s); + return s; + } + } private static Set<NewDocumentType.Name> convertDocumentReferencesToNames(Optional<DocumentReferences> documentReferences) { @@ -468,9 +575,9 @@ public class DocumentModelBuilder { return Set.of(); } return documentReferences.get().referenceMap().values().stream() - .map(documentReference -> documentReference.targetSearch().getDocument()) - .map(documentType -> new NewDocumentType.Name(documentType.getName())) - .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + .map(documentReference -> documentReference.targetSearch().getDocument()) + .map(documentType -> new NewDocumentType.Name(documentType.getName())) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); } private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { @@ -480,62 +587,6 @@ public class DocumentModelBuilder { return Collections.unmodifiableSet(importedFields.fields().keySet()); } - private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { - for (Field f : fields) { - DataType type = f.getDataType(); - if (testAddType(dt, type)) { - extractNestedTypes(dt, type); - addType(dt, type); - } - } - } - - private static void extractNestedTypes(NewDocumentType dt, DataType type) { - if (type instanceof StructDataType) { - StructDataType tmp = (StructDataType) type; - extractDataTypesFromFields(dt, tmp.getFieldsThisTypeOnly()); - } else if (type instanceof DocumentType) { - throw new IllegalArgumentException("Can not handle nested document definitions. In document type '" + dt.getName().toString() + - "', we can not define document type '" + type.toString()); - } else if (type instanceof CollectionDataType) { - CollectionDataType tmp = (CollectionDataType) type; - extractNestedTypes(dt, tmp.getNestedType()); - addType(dt, tmp.getNestedType()); - } else if (type instanceof MapDataType) { - MapDataType tmp = (MapDataType) type; - extractNestedTypes(dt, tmp.getKeyType()); - extractNestedTypes(dt, tmp.getValueType()); - addType(dt, tmp.getKeyType()); - addType(dt, tmp.getValueType()); - } else if (type instanceof TemporaryAnnotationReferenceDataType) { - throw new IllegalArgumentException(type.toString()); - } - } - - private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); } - - private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); } - - private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { - DataType oldType = dt.getDataTypeRecursive(type.getId()); - if (oldType == null) { - if ( ! dryRun) { - dt.add(type); - } - return true; - } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { - StructDataType s = (StructDataType) type; - StructDataType os = (StructDataType) oldType; - if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { - if ( ! dryRun) { - dt.replace(type); - } - return true; - } - } - return false; - } - public static class RetryLaterException extends IllegalArgumentException { public RetryLaterException(String message) { super(message); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 5e5623e2319..4a5a858f828 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -20,47 +20,8 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { for (SDDocumentType type : sdoc.getAllTypes()) { if (type.isStruct()) { apply(type); - copyFields(type, sdoc); } } } - @SuppressWarnings("deprecation") - private void copyFields(SDDocumentType structType, SDDocumentType sdoc) { - //find all fields in OTHER types that have this type: - List<SDDocumentType> list = new ArrayList<>(); - list.add(sdoc); - list.addAll(sdoc.getTypes()); - for (SDDocumentType anyType : list) { - Iterator<Field> fields = anyType.fieldIterator(); - while (fields.hasNext()) { - SDField field = (SDField) fields.next(); - maybePopulateField(sdoc, field, structType); - } - } - } - - private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) { - DataType structUsedByField = field.getFirstStructRecursive(); - if (structUsedByField == null) { - return; - } - if (structUsedByField.getName().equals(structType.getName())) { - //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); - } - } - - public void processSchemaFields(Schema schema) { - var sdoc = schema.getDocument(); - if (sdoc == null) return; - for (SDDocumentType type : sdoc.getAllTypes()) { - if (type.isStruct()) { - for (SDField field : schema.allExtraFields()) { - maybePopulateField(sdoc, field, type); - } - } - } - } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java index ba34045e7de..848631f1739 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -81,7 +81,10 @@ public class SDDocumentTypeOrderer { } else { inherited = createdSDTypes.get(name); - if (inherited == null) throw new IllegalArgumentException("Document type '" + name + "' not found"); + if (inherited == null) { + throw new IllegalArgumentException("document " + type.getName() + + " inherits from unavailable document " + name); + } process(inherited, inherited); } type.inherit(inherited); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index e627f06ad42..b87bdd8907e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -18,7 +18,6 @@ import com.yahoo.searchdefinition.Schema; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -37,8 +36,8 @@ import java.util.Set; public class SDDocumentType implements Cloneable, Serializable { public static final SDDocumentType VESPA_DOCUMENT; - private final Map<DataTypeName, SDDocumentType> inheritedTypes = new HashMap<>(); - private final Map<NewDocumentType.Name, SDDocumentType> ownedTypes = new HashMap<>(); + private final Map<DataTypeName, SDDocumentType> inheritedTypes = new LinkedHashMap<>(); + private final Map<NewDocumentType.Name, SDDocumentType> ownedTypes = new LinkedHashMap<>(); private final AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry(); private DocumentType docType; private DataType structType; @@ -269,8 +268,8 @@ public class SDDocumentType implements Cloneable, Serializable { return field; } - public Field addField(String string, DataType dataType, boolean header, int code) { - SDField field = new SDField(this, string, code, dataType, header); + public Field addField(String fName, DataType dataType, boolean header, int code) { + SDField field = new SDField(this, fName, code, dataType); addField(field); return field; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index d9bdf5dc917..8263352e87f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -9,6 +9,7 @@ import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; @@ -110,8 +111,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Struct fields defined in this field */ private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0); - /** The document that this field was declared in, or null*/ - private SDDocumentType ownerDocType = null; + /** The document that this field was declared in, or null */ + private SDDocumentType repoDocType = null; /** The aliases declared for this field. May pertain to indexes or attributes */ private final Map<String, String> aliasToName = new HashMap<>(); @@ -129,25 +130,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * @param name the name of the field * @param dataType the datatype of the field */ - protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { super(name, id, dataType); - populate(populate, repo, name, dataType); + this.repoDocType = repo; + populate(name, dataType); } - public SDField(SDDocumentType repo, String name, int id, DataType dataType) { - this(repo, name, id, dataType, true); + public SDField(String name, DataType dataType) { + this(null, name, dataType); } /** Creates a new field */ - public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { - super(name, dataType); - populate(populate, repo, name, dataType); + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name, dataType, null); } /** Creates a new field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { - super(name, dataType, owner == null ? null : owner.getDocumentType()); - populate(populate, repo, name, dataType); + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + this(repo, name, dataType, owner, null, 0); } /** @@ -158,27 +158,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * @param owner the owning document (used to check for id collisions) * @param fieldMatching the matching object to set for the field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, - Matching fieldMatching, boolean populate, int recursion) { + protected SDField(SDDocumentType repo, + String name, + DataType dataType, + SDDocumentType owner, + Matching fieldMatching, + int recursion) + { super(name, dataType, owner == null ? null : owner.getDocumentType()); + this.repoDocType = repo; + this.structFieldDepth = recursion; if (fieldMatching != null) this.setMatching(fieldMatching); - populate(populate, repo, name, dataType, fieldMatching, recursion); - } - - public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name, dataType, true); + populate(name, dataType); } - public SDField(String name, DataType dataType) { - this(null, name, dataType); - } - - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { - populate(populate, repo, name, dataType, null, 0); - } + private int structFieldDepth = 0; - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { + private void populate(String name, DataType dataType) { if (dataType instanceof TensorDataType) { TensorType type = ((TensorDataType)dataType).getTensorType(); if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) @@ -186,13 +183,13 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, ": Dense tensor dimensions must have a size"); addQueryCommand("type " + type); } + else if (dataType instanceof WeightedSetDataType) { + var nested = ((WeightedSetDataType) dataType).getNestedType().getName(); + addQueryCommand("type WeightedSet<" + nested + ">"); + } else { addQueryCommand("type " + dataType.getName()); } - if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, recursion); - populateWithStructMatching(repo, dataType, fieldMatching); - } } public void setIsExtraField(boolean isExtra) { @@ -268,17 +265,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } + private boolean doneStructFields = false; + @SuppressWarnings("deprecation") - public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { - DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) return; + private void actuallyMakeStructFields() { + if (doneStructFields) return; + if (getFirstStructOrMapRecursive() == null) { + doneStructFields = true; + return; + } + var sdoc = repoDocType; + var dataType = getDataType(); java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> { if (structFields.containsKey(fieldName)) return; - String subName = name.concat(".").concat(fieldName); - var subField = new SDField(sdoc, subName, fieldType, - ownerDocType, new Matching(), - true, recursion + 1); + String subName = getName().concat(".").concat(fieldName); + var subField = new SDField(sdoc, subName, fieldType, null, + null, structFieldDepth + 1); structFields.put(fieldName, subField); }; @@ -287,15 +290,16 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, supplyStructField.accept("key", mdt.getKeyType()); supplyStructField.accept("value", mdt.getValueType()); } else { - if (recursion >= 10) return; + if (structFieldDepth >= 10) { + // too risky, infinite recursion + doneStructFields = true; + return; + } if (dataType instanceof CollectionDataType) { dataType = ((CollectionDataType)dataType).getNestedType(); } - if (dataType instanceof TemporaryStructuredDataType) { - SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (subType == null) { - throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); - } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (dataType instanceof TemporaryStructuredDataType && subType != null) { for (Field field : subType.fieldSet()) { supplyStructField.accept(field.getName(), field.getDataType()); } @@ -305,36 +309,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, supplyStructField.accept(field.getName(), field.getDataType()); } } - } - } - - public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) { - if (sdoc == null) return; - if (superFieldMatching == null) return; - DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) return; - - if (dataType instanceof MapDataType) { - // old code here would never do anything useful, should we do something here? - return; - } else { - if (dataType instanceof CollectionDataType) { - dataType = ((CollectionDataType)dataType).getNestedType(); + if ((subType == null) && (structFields.size() > 0)) { + throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + + this + " [" + getDataType() + getDataType().getClass() + + "] with " + structFields.size() + " struct fields"); } - if (dataType instanceof StructDataType) { - SDDocumentType subType = sdoc.getType(dataType.getName()); - if (subType == null) { - throw new IllegalArgumentException("Could not find struct " + dataType.getName()); - } + // populate struct fields with matching + if (subType != null) { for (Field f : subType.fieldSet()) { if (f instanceof SDField) { SDField field = (SDField) f; Matching subFieldMatching = new Matching(); - subFieldMatching.merge(superFieldMatching); + subFieldMatching.merge(this.matching); subFieldMatching.merge(field.getMatching()); SDField subField = structFields.get(field.getName()); if (subField != null) { - subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching); + // we just made this with no matching, so nop: + // subFieldMatching.merge(subField.getMatching()); subField.setMatching(subFieldMatching); } } else { @@ -343,8 +334,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } } + doneStructFields = true; } + private Matching matchingForStructFields = null; + public void addOperation(FieldOperation op) { pendingOperations.add(op); } @@ -717,7 +711,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Returns list of static struct fields */ @Override - public Collection<SDField> getStructFields() { return structFields.values(); } + public Collection<SDField> getStructFields() { + actuallyMakeStructFields(); + return structFields.values(); + } /** * Returns a struct field defined in this field, @@ -726,6 +723,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ @Override public SDField getStructField(String name) { + actuallyMakeStructFields(); if (name.contains(".")) { String superFieldName = name.substring(0,name.indexOf(".")); String subFieldName = name.substring(name.indexOf(".")+1); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 4ced104fa55..8c17b607f94 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -8,12 +8,12 @@ import com.yahoo.document.DataType; */ public class TemporarySDField extends SDField { - public TemporarySDField(String name, DataType dataType, SDDocumentType owner) { - super(owner, name, dataType, owner, false); + public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + super(repo, name, dataType, owner); } - public TemporarySDField(String name, DataType dataType) { - super(null, name, dataType, false); + public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { + super(repo, name, dataType); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java index fe3ac11af27..a5f5f961ab5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java @@ -24,6 +24,8 @@ public class IndexingOperation implements FieldOperation { this.script = script; } + public ScriptExpression getScript() { return script; } + public void apply(SDField field) { field.setIndexingScript(script); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java index e7423b17830..caeebd65f4f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -138,6 +138,7 @@ public class ConvertParsedFields { if (indexing.isPresent()) { field.setIndexingScript(indexing.get().script()); } + parsed.getWeight().ifPresent(value -> field.setWeight(value)); parsed.getStemming().ifPresent(value -> field.setStemming(value)); parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); for (var attribute : parsed.getAttributes()) { @@ -290,13 +291,12 @@ public class ConvertParsedFields { schema.addIndex(index); } - SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) { + SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) { // TODO - can we cleanup this mess var structProxy = new SDDocumentType(parsed.name(), schema); - structProxy.setStruct(context.resolveStruct(parsed)); for (var parsedField : parsed.getFields()) { var fieldType = context.resolveType(parsedField.getType()); - var field = new SDField(structProxy, parsedField.name(), fieldType); + var field = new SDField(document, parsedField.name(), fieldType); convertCommonFieldSettings(field, parsedField); structProxy.addField(field); if (parsedField.hasIdOverride()) { @@ -306,7 +306,17 @@ public class ConvertParsedFields { for (String inherit : parsed.getInherited()) { structProxy.inherit(new DataTypeName(inherit)); } + structProxy.setStruct(context.resolveStruct(parsed)); return structProxy; } + void convertAnnotation(Schema schema, SDDocumentType document, ParsedAnnotation parsed) { + var annType = context.resolveAnnotation(parsed.name()); + var payload = parsed.getStruct(); + if (payload.isPresent()) { + var structProxy = convertStructDeclaration(schema, document, payload.get()); + document.addType(structProxy); + } + document.addAnnotation(annType); + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java index e67c1ac8275..b22c33c527d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java @@ -10,6 +10,7 @@ import com.yahoo.document.PositionDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import java.util.ArrayList; import java.util.Collection; @@ -63,22 +64,64 @@ public class ConvertParsedTypes { } for (var annotation : doc.getAnnotations()) { String annId = doc.name() + "->" + annotation.name(); - var at = new AnnotationType(annotation.name()); + var at = new SDAnnotationType(annotation.name()); annotationsFromSchemas.put(annId, at); - var withStruct = annotation.getStruct(); - if (withStruct.isPresent()) { - var sn = withStruct.get().name(); - var dt = new StructDataType(sn); - String structId = doc.name() + "->" + sn; - structsFromSchemas.put(structId, dt); - } } } } + Map<ParsedAnnotation, ParsedStruct> filledAnnotations = new HashMap<>(); + + ParsedStruct fillAnnotationStruct(ParsedAnnotation annotation) { + if (filledAnnotations.containsKey(annotation)) { + return filledAnnotations.get(annotation); + } + var doc = annotation.getOwnerDoc(); + for (String inherit : annotation.getInherited()) { + var parent = findParsedAnnotation(doc, inherit); + var parentStruct = fillAnnotationStruct(parent); + if (parentStruct == null) { + continue; + } + var myStruct = annotation.getStruct().orElse + (new ParsedStruct("annotation." + annotation.name())); + myStruct.inherit(parentStruct.name()); + annotation.setStruct(myStruct); + } + var withStruct = annotation.getStruct(); + if (withStruct.isPresent()) { + ParsedStruct struct = withStruct.get(); + String structId = doc.name() + "->" + struct.name(); + var toFill = structsFromSchemas.computeIfAbsent(structId, k -> new StructDataType(struct.name())); + for (ParsedField field : struct.getFields()) { + var t = resolveFromContext(field.getType(), doc); + var f = field.hasIdOverride() + ? new com.yahoo.document.Field(field.name(), field.idOverride(), t) + : new com.yahoo.document.Field(field.name(), t); + toFill.addField(f); + } + var at = findAnnotationFromParsed(annotation); + at.setDataType(toFill); + filledAnnotations.put(annotation, struct); + } else { + filledAnnotations.put(annotation, null); + } + return withStruct.orElse(null); + } + private void fillDataTypes() { for (var schema : orderedInput) { var doc = schema.getDocument(); + for (var annotation : doc.getAnnotations()) { + var at = findAnnotationFromParsed(annotation); + for (String inherit : annotation.getInherited()) { + var parent = findAnnotationFromSchemas(inherit, doc); + at.inherit(parent); + } + } + for (var annotation : doc.getAnnotations()) { + fillAnnotationStruct(annotation); + } for (var struct : doc.getStructs()) { String structId = doc.name() + "->" + struct.name(); var toFill = structsFromSchemas.get(structId); @@ -99,29 +142,20 @@ public class ConvertParsedTypes { } for (String inherit : struct.getInherited()) { var parent = findStructFromSchemas(inherit, doc); - toFill.inherit(parent); - } - } - for (var annotation : doc.getAnnotations()) { - String annId = doc.name() + "->" + annotation.name(); - var at = annotationsFromSchemas.get(annId); - var withStruct = annotation.getStruct(); - if (withStruct.isPresent()) { - ParsedStruct struct = withStruct.get(); - String structId = doc.name() + "->" + struct.name(); - var toFill = structsFromSchemas.get(structId); - for (ParsedField field : struct.getFields()) { - var t = resolveFromContext(field.getType(), doc); - var f = field.hasIdOverride() - ? new com.yahoo.document.Field(field.name(), field.idOverride(), t) - : new com.yahoo.document.Field(field.name(), t); - toFill.addField(f); + // ensure a nice, compatible exception message + for (var field : toFill.getFields()) { + if (parent.hasField(field)) { + for (var base : parent.getInheritedTypes()) { + if (base.hasField(field)) { + parent = base; + } + } + throw new IllegalArgumentException + ("In document " + doc.name() + ": struct " + struct.name() + + " cannot inherit from " + parent.getName() + " and redeclare field " + field.getName()); + } } - at.setDataType(toFill); - } - for (String inherit : annotation.getInherited()) { - var parent = findAnnotationFromSchemas(inherit, doc); - at.inherit(parent); + toFill.inherit(parent); } } var docToFill = documentsFromSchemas.get(doc.name()); @@ -140,6 +174,7 @@ public class ConvertParsedTypes { fieldSets.put("[document]", inDocFields); for (var extraField : schema.getFields()) { String name = extraField.name(); + if (docToFill.hasField(name)) continue; var t = resolveFromContext(extraField.getType(), doc); var f = new com.yahoo.document.Field(name, t); docToFill.addField(f); @@ -167,7 +202,7 @@ public class ConvertParsedTypes { private AnnotationType findAnnotationFromSchemas(String name, ParsedDocument context) { var resolved = findParsedAnnotation(context, name); - String annotationId = resolved.getOwner() + "->" + resolved.name(); + String annotationId = resolved.getOwnerName() + "->" + resolved.name(); var annotation = annotationsFromSchemas.get(annotationId); if (annotation == null) { throw new IllegalArgumentException("no annotation named " + name + " in context " + context); @@ -175,6 +210,15 @@ public class ConvertParsedTypes { return annotation; } + private AnnotationType findAnnotationFromParsed(ParsedAnnotation resolved) { + String annotationId = resolved.getOwnerName() + "->" + resolved.name(); + var annotation = annotationsFromSchemas.get(annotationId); + if (annotation == null) { + throw new IllegalArgumentException("no annotation " + resolved.name() + " in " + resolved.getOwnerName()); + } + return annotation; + } + private ParsedStruct findParsedStruct(ParsedDocument doc, String name) { ParsedStruct found = doc.getStruct(name); if (found != null) return found; @@ -299,6 +343,9 @@ public class ConvertParsedTypes { } return r; } + public AnnotationType resolveAnnotation(String name) { + return findAnnotationFromSchemas(name, context); + } TypeResolver(ParsedDocument context) { this.context = context; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 332a2153516..316170893ee 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -33,7 +33,9 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -122,6 +124,8 @@ public class ConvertSchemaCollection { typeConverter.convert(true); } + private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap(); + public List<Schema> convertToSchemas() { typeConverter = new ConvertParsedTypes(orderedInput, docMan); typeConverter.convert(false); @@ -145,34 +149,21 @@ public class ConvertSchemaCollection { return resultList; } - private void convertAnnotation(Schema schema, SDDocumentType document, ParsedAnnotation parsed, ConvertParsedFields fieldConverter) { - var type = new SDAnnotationType(parsed.name()); - for (String inherit : parsed.getInherited()) { - type.inherit(inherit); - } - var payload = parsed.getStruct(); - if (payload.isPresent()) { - var struct = fieldConverter.convertStructDeclaration(schema, payload.get()); - type = new SDAnnotationType(parsed.name(), struct, type.getInherits()); - // WTF? - struct.setStruct(null); - } - document.addAnnotation(type); - } - private void convertDocument(Schema schema, ParsedDocument parsed, ConvertParsedFields fieldConverter) { SDDocumentType document = new SDDocumentType(parsed.name()); for (String inherit : parsed.getInherited()) { - document.inherit(new DataTypeName(inherit)); + var parent = convertedDocuments.get(inherit); + assert(parent != null); + document.inherit(parent); } for (var struct : parsed.getStructs()) { - var structProxy = fieldConverter.convertStructDeclaration(schema, struct); + var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); document.addType(structProxy); } for (var annotation : parsed.getAnnotations()) { - convertAnnotation(schema, document, annotation, fieldConverter); + fieldConverter.convertAnnotation(schema, document, annotation); } for (var field : parsed.getFields()) { var sdf = fieldConverter.convertDocumentField(schema, document, field); @@ -180,6 +171,7 @@ public class ConvertSchemaCollection { document.setFieldId(sdf, field.idOverride()); } } + convertedDocuments.put(parsed.name(), document); schema.addDocument(document); } @@ -239,7 +231,7 @@ public class ConvertSchemaCollection { if (parsed.hasStemming()) { schema.setStemming(parsed.getStemming()); } - schema.enableRawAsBase64(parsed.getRawAsBase64()); + parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); var typeContext = typeConverter.makeContext(parsed.getDocument()); var fieldConverter = new ConvertParsedFields(typeContext); convertDocument(schema, parsed.getDocument(), fieldConverter); @@ -258,6 +250,9 @@ public class ConvertSchemaCollection { for (var fieldSet : parsed.getFieldSets()) { convertFieldSet(schema, fieldSet); } + if (documentsOnly) { + return; // skip ranking-only content, not used for document type generation + } for (var rankingConstant : parsed.getRankingConstants()) { schema.rankingConstants().add(rankingConstant); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java index 536caf55111..23b5195486b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java @@ -12,7 +12,7 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +27,7 @@ public class IntermediateCollection { private final DeployLogger deployLogger; private final ModelContext.Properties modelProperties; - private Map<String, ParsedSchema> parsedSchemas = new HashMap<>(); + private Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>(); IntermediateCollection() { this.deployLogger = new BaseDeployLogger(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java index f22d370b1d8..096effa6810 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java @@ -14,7 +14,7 @@ class ParsedAnnotation extends ParsedBlock { private ParsedStruct wrappedStruct = null; private final List<String> inherited = new ArrayList<>(); - private String ownedBy = null; + private ParsedDocument ownedBy = null; ParsedAnnotation(String name) { super(name, "annotation"); @@ -22,12 +22,16 @@ class ParsedAnnotation extends ParsedBlock { public List<String> getInherited() { return List.copyOf(inherited); } public Optional<ParsedStruct> getStruct() { return Optional.ofNullable(wrappedStruct); } - public String getOwner() { return ownedBy; } + public ParsedDocument getOwnerDoc() { return ownedBy; } + public String getOwnerName() { return ownedBy.name(); } void setStruct(ParsedStruct struct) { this.wrappedStruct = struct; } + void inherit(String other) { inherited.add(other); } - void tagOwner(String owner) { + + void tagOwner(ParsedDocument owner) { verifyThat(ownedBy == null, "already owned by", ownedBy); this.ownedBy = owner; + getStruct().ifPresent(s -> s.tagOwner(owner.name())); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index ed975238067..679ed72c6f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -65,8 +65,7 @@ public class ParsedDocument extends ParsedBlock { String annName = annotation.name(); verifyThat(! docAnnotations.containsKey(annName), "already has annotation", annName); docAnnotations.put(annName, annotation); - annotation.tagOwner(name()); - annotation.getStruct().ifPresent(s -> s.tagOwner(name())); + annotation.tagOwner(this); } public String toString() { return "document " + name(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java index 5bcfa841bae..ca876997dc6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java @@ -58,6 +58,7 @@ class ParsedField extends ParsedBlock { List<String> getQueryCommands() { return List.copyOf(queryCommands); } String lookupAliasedFrom(String alias) { return aliases.get(alias); } ParsedMatchSettings matchSettings() { return this.matchInfo; } + Optional<Integer> getWeight() { return Optional.ofNullable(weight); } Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); } Optional<String> getNormalizing() { return Optional.ofNullable(normalizing); } Optional<ParsedIndexingOp> getIndexing() { return Optional.ofNullable(indexingOp); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index 0004094e1c2..599dd6e2a7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * This class holds the extracted information after parsing @@ -30,7 +31,7 @@ public class ParsedSchema extends ParsedBlock { } private boolean documentWithoutSchema = false; - private boolean rawAsBase64 = false; // TODO Vespa 8 flip default + private Boolean rawAsBase64 = null; private ParsedDocument myDocument = null; private Stemming defaultStemming = null; private final List<ImportedField> importedFields = new ArrayList<>(); @@ -53,7 +54,7 @@ public class ParsedSchema extends ParsedBlock { } boolean getDocumentWithoutSchema() { return documentWithoutSchema; } - boolean getRawAsBase64() { return rawAsBase64; } + Optional<Boolean> getRawAsBase64() { return Optional.ofNullable(rawAsBase64); } boolean hasDocument() { return myDocument != null; } ParsedDocument getDocument() { return myDocument; } boolean hasStemming() { return defaultStemming != null; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index 3aed90a58e1..d04277706a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -46,6 +46,7 @@ class ParsedType { case "raw": return Variant.BUILTIN; case "tag": return Variant.BUILTIN; case "position": return Variant.POSITION; + case "float16": return Variant.BUILTIN; } return Variant.UNKNOWN; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 51defffa00b..0be48d1fd25 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -70,7 +70,7 @@ public class AddExtraFieldsToDocument extends Processor { if (docField == null) { ImmutableSDField existingField = schema.getField(field.getName()); if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType(), true); + SDField newField = new SDField(document, field.getName(), field.getDataType()); newField.setIsExtraField(true); document.addField(newField); } else if (!existingField.isImportedField()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index 0bb1b7da769..d7882c7f8fb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -10,6 +10,7 @@ import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.GeoPos; +import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -35,8 +36,11 @@ import java.util.logging.Level; */ public class CreatePositionZCurve extends Processor { + private final SDDocumentType repo; + public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); + this.repo = schema.getDocument(); } private boolean useV8GeoPositions = false; @@ -105,7 +109,7 @@ public class CreatePositionZCurve extends Processor { "' already created."); } boolean isArray = inputField.getDataType() instanceof ArrayDataType; - SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); + SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY : Attribute.CollectionType.SINGLE); attribute.setPosition(true); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index 84dc6d369fc..7397f9a289c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -61,7 +61,7 @@ public class UriHack extends Processor { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(partName, generatedType); + SDField partField = new SDField(schema.getDocument(), partName, generatedType); partField.setIndexStructureField(uriField.doesIndexing()); partField.setRankType(uriField.getRankType()); partField.setStemming(Stemming.NONE); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 9b4b3eba3a7..4dbeaef17ac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -11,9 +11,11 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; @@ -28,7 +30,7 @@ import java.util.Set; public class DocumentManager { private boolean useV8GeoPositions = false; - private boolean useV8DocManagerCfg = false; + private boolean useV8DocManagerCfg = true; public DocumentManager useV8GeoPositions(boolean value) { this.useV8GeoPositions = value; @@ -262,6 +264,13 @@ public class DocumentManager { } } + static private <T> List<T> sortedList(Collection<T> unsorted, Comparator<T> cmp) { + var list = new ArrayList<T>(); + list.addAll(unsorted); + list.sort(cmp); + return list; + } + private void docTypeBuild(NewDocumentType documentType, DocumentmanagerConfig.Builder builder, IdxMap indexMap) { DocumentmanagerConfig.Doctype.Builder db = new DocumentmanagerConfig.Doctype.Builder(); db. @@ -274,11 +283,14 @@ public class DocumentManager { db.inherits(b -> b.idx(indexMap.idxOf(inherited))); } docTypeBuildAnyType(documentType.getHeader(), db, indexMap); - for (DataType dt : documentType.getAllTypes().getTypes()) { + + for (DataType dt : sortedList(documentType.getAllTypes().getTypes(), + (a,b) -> a.getName().compareTo(b.getName()))) { docTypeBuildAnyType(dt, db, indexMap); } - for (AnnotationType annotation : documentType.getAnnotations()) { - docTypeBuildAnnotationType(annotation, db, indexMap); + for (AnnotationType ann : sortedList(documentType.getAnnotations(), + (a,b) -> a.getName().compareTo(b.getName()))) { + docTypeBuildAnnotationType(ann, db, indexMap); } builder.doctype(db); indexMap.setDone(documentType); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index ac1b92e287f..1bd80988d58 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -42,6 +42,13 @@ public class DocumentTypes { } } + static private <T> List<T> sortedList(Collection<T> unsorted, Comparator<T> cmp) { + var list = new ArrayList<T>(); + list.addAll(unsorted); + list.sort(cmp); + return list; + } + private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) { if (documentType == VespaDocumentType.INSTANCE) { return; @@ -56,10 +63,10 @@ public class DocumentTypes { db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); markAsBuilt(built, inherited.getAllTypes()); } - for (DataType dt : documentType.getTypes()) { + for (DataType dt : sortedList(documentType.getTypes(), (a,b) -> a.getName().compareTo(b.getName()))) { buildConfig(dt, db, built); } - for (AnnotationType annotation : documentType.getAnnotations()) { + for (AnnotationType annotation : sortedList(documentType.getAnnotations(), (a,b) -> a.getName().compareTo(b.getName()))) { DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder(); db.annotationtype(atb); buildConfig(annotation, atb); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 8390cc59b6f..ac99bee93ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -108,6 +108,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public static final Logger log = Logger.getLogger(VespaModel.class.getName()); private final Version version; + private final Version wantedNodeVersion; private final ConfigModelRepo configModelRepo = new ConfigModelRepo(); private final AllocatedHosts allocatedHosts; @@ -170,6 +171,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri throws IOException, SAXException { super("vespamodel"); version = deployState.getVespaVersion(); + wantedNodeVersion = deployState.getWantedNodeVespaVersion(); fileReferencesRepository = new FileReferencesRepository(deployState.getFileRegistry()); rankingConstants = new RankingConstants(deployState.getFileRegistry(), Optional.empty()); validationOverrides = deployState.validationOverrides(); @@ -407,6 +409,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return version; } + @Override + public Version wantedNodeVersion() { + return wantedNodeVersion; + } + /** * Resolves config of the given type and config id, by first instantiating the correct {@link com.yahoo.config.ConfigInstance.Builder}, * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 7447eee5cec..ba955f071b2 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -958,10 +958,7 @@ void indexingOperation(ParsedField field, boolean multiLine) : { } { { IndexingOperation oldOp = newIndexingOperation(multiLine); - // TODO conversion via SDField is very ugly - SDField tmpField = new SDField("temp", com.yahoo.document.DataType.STRING); - oldOp.apply(tmpField); - ParsedIndexingOp newOp = new ParsedIndexingOp(tmpField.getIndexingScript()); + ParsedIndexingOp newOp = new ParsedIndexingOp(oldOp.getScript()); field.setIndexingOperation(newOp); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 1a0b518bbb2..e90df2776e0 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -639,7 +639,7 @@ void field(SDDocumentType document, Schema schema) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, document); + field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document); } lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE> { @@ -668,7 +668,7 @@ void fieldSet(Schema schema) : | ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); } | - ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } + ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } )(<NL>)*)+ <RBRACE> { @@ -698,11 +698,13 @@ void annotationOutside(Schema schema) : <ANNOTATION> name = identifier() { type = new SDAnnotationType(name.trim()); + if (schema.getDocument() == null) { + throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); + } } [ inheritsAnnotation(type) (<NL>)* ] - lbrace() (type = annotationBody(schema, type)) <RBRACE> + lbrace() (type = annotationBody(schema, schema.getDocument(), type)) <RBRACE> { - if (schema.getDocument()==null) throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); schema.addAnnotation(type); } } @@ -723,7 +725,7 @@ void annotation(Schema schema, SDDocumentType document) : type = new SDAnnotationType(name.trim()); } [ inheritsAnnotation(type) (<NL>)* ] - lbrace() (type = annotationBody(schema, type)) <RBRACE> + lbrace() (type = annotationBody(schema, document, type)) <RBRACE> { document.addAnnotation(type); } @@ -737,12 +739,12 @@ void annotation(Schema schema, SDDocumentType document) : * @param type the type being built * @return a modified or new AnnotationType instance */ -SDAnnotationType annotationBody(Schema schema, SDAnnotationType type) : +SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) : { SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema); } { - (structFieldDefinition(struct) (<NL>)*)* + (structFieldDefinition(repo, struct) (<NL>)*)* { if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField. type = new SDAnnotationType(type.getName(), struct, type.getInherits()); @@ -809,9 +811,13 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : SDDocumentType struct; } { - ( <STRUCT> name = identifier() (<NL>)* { struct = new SDDocumentType(name, schema); } + ( <STRUCT> name = identifier() (<NL>)* { + if (repo == null) + throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); + struct = new SDDocumentType(name, schema); + } [ inheritsDocument(struct) (<NL>)* ] - lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> ) + lbrace() (structFieldDefinition(repo, struct) (<NL>)*)* <RBRACE> ) { try { docMan.getDataType(name); @@ -819,7 +825,6 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : } catch (IllegalArgumentException e) { // empty } - if (repo==null) throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName()); DataType stype = sdtype != null ? sdtype.getStruct() @@ -828,7 +833,7 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : return struct; } } - + /** * This rule consumes a data type block from within a field element. * @@ -921,7 +926,7 @@ DataType wildCardType() : * * @param struct The struct to modify. */ -void structFieldDefinition(SDDocumentType struct) : +void structFieldDefinition(SDDocumentType document, SDDocumentType struct) : { String name; SDField field; @@ -932,7 +937,7 @@ void structFieldDefinition(SDDocumentType struct) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, struct); + field = new TemporarySDField(document, name, type, struct); struct.addField(field); } lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> { diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index 8b93e3a4665..7a983693c22 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -1,251 +1,282 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id -794985308 -datatype[1].arraytype[0].datatype 1707615575 -datatype[2].id 1707615575 -datatype[2].arraytype[0].datatype -1486737430 -datatype[3].id 1416345047 -datatype[3].arraytype[0].datatype -372512406 -datatype[4].id 69621385 -datatype[4].arraytype[0].datatype 339965458 -datatype[5].id 49942803 -datatype[5].arraytype[0].datatype 16 -datatype[6].id -1245117006 -datatype[6].arraytype[0].datatype 0 -datatype[7].id 759956026 -datatype[7].arraytype[0].datatype -2092985853 -datatype[8].id -1244829667 -datatype[8].arraytype[0].datatype 109267174 -datatype[9].id -1486737430 -datatype[9].arraytype[0].datatype 2 -datatype[10].id -372512406 -datatype[10].maptype[0].keytype 0 -datatype[10].maptype[0].valtype 1707615575 -datatype[11].id 2138385264 -datatype[11].maptype[0].keytype 0 -datatype[11].maptype[0].valtype 5 -datatype[12].id -389833101 -datatype[12].maptype[0].keytype 0 -datatype[12].maptype[0].valtype 294108848 -datatype[13].id -1715531035 -datatype[13].maptype[0].keytype 0 -datatype[13].maptype[0].valtype 4 -datatype[14].id 1901258752 -datatype[14].maptype[0].keytype 0 -datatype[14].maptype[0].valtype -2092985853 -datatype[15].id 435886609 -datatype[15].maptype[0].keytype 2 -datatype[15].maptype[0].valtype -1245117006 -datatype[16].id 2125154557 -datatype[16].maptype[0].keytype 2 -datatype[16].maptype[0].valtype 1 -datatype[17].id -1584287606 -datatype[17].maptype[0].keytype 2 -datatype[17].maptype[0].valtype 0 -datatype[18].id -1865479609 -datatype[18].maptype[0].keytype 2 -datatype[18].maptype[0].valtype 4 -datatype[19].id 339965458 -datatype[19].maptype[0].keytype 2 -datatype[19].maptype[0].valtype 2 -datatype[20].id 1328286588 -datatype[20].weightedsettype[0].datatype 2 -datatype[20].weightedsettype[0].createifnonexistant false -datatype[20].weightedsettype[0].removeifzero false -datatype[21].id 2065577986 -datatype[21].weightedsettype[0].datatype 2 -datatype[21].weightedsettype[0].createifnonexistant true -datatype[21].weightedsettype[0].removeifzero false -datatype[22].id 2125328771 -datatype[22].weightedsettype[0].datatype 2 -datatype[22].weightedsettype[0].createifnonexistant false -datatype[22].weightedsettype[0].removeifzero true -datatype[23].id 294108848 -datatype[23].structtype[0].name "folder" -datatype[23].structtype[0].version 0 -datatype[23].structtype[0].compresstype NONE -datatype[23].structtype[0].compresslevel 0 -datatype[23].structtype[0].compressthreshold 95 -datatype[23].structtype[0].compressminsize 800 -datatype[23].structtype[0].field[0].name "Version" -datatype[23].structtype[0].field[0].datatype 0 -datatype[23].structtype[0].field[0].detailedtype "" -datatype[23].structtype[0].field[1].name "Name" -datatype[23].structtype[0].field[1].datatype 2 -datatype[23].structtype[0].field[1].detailedtype "" -datatype[23].structtype[0].field[2].name "FlagsCounter" -datatype[23].structtype[0].field[2].datatype -1865479609 -datatype[23].structtype[0].field[2].detailedtype "" -datatype[23].structtype[0].field[3].name "anotherfolder" -datatype[23].structtype[0].field[3].datatype 294108848 -datatype[23].structtype[0].field[3].detailedtype "" -datatype[24].id -2092985853 -datatype[24].structtype[0].name "mystruct" -datatype[24].structtype[0].version 0 -datatype[24].structtype[0].compresstype NONE -datatype[24].structtype[0].compresslevel 0 -datatype[24].structtype[0].compressthreshold 95 -datatype[24].structtype[0].compressminsize 800 -datatype[24].structtype[0].field[0].name "bytearr" -datatype[24].structtype[0].field[0].datatype 49942803 -datatype[24].structtype[0].field[0].detailedtype "" -datatype[24].structtype[0].field[1].name "mymap" -datatype[24].structtype[0].field[1].datatype 339965458 -datatype[24].structtype[0].field[1].detailedtype "" -datatype[24].structtype[0].field[2].name "title" -datatype[24].structtype[0].field[2].datatype 2 -datatype[24].structtype[0].field[2].detailedtype "" -datatype[24].structtype[0].field[3].name "structfield" -datatype[24].structtype[0].field[3].datatype 2 -datatype[24].structtype[0].field[3].detailedtype "" -datatype[25].id 109267174 -datatype[25].structtype[0].name "sct" -datatype[25].structtype[0].version 0 -datatype[25].structtype[0].compresstype NONE -datatype[25].structtype[0].compresslevel 0 -datatype[25].structtype[0].compressthreshold 95 -datatype[25].structtype[0].compressminsize 800 -datatype[25].structtype[0].field[0].name "s1" -datatype[25].structtype[0].field[0].datatype 2 -datatype[25].structtype[0].field[0].detailedtype "" -datatype[25].structtype[0].field[1].name "s2" -datatype[25].structtype[0].field[1].datatype 2 -datatype[25].structtype[0].field[1].detailedtype "" -datatype[26].id 1328581348 -datatype[26].structtype[0].name "types.header" -datatype[26].structtype[0].version 0 -datatype[26].structtype[0].compresstype NONE -datatype[26].structtype[0].compresslevel 0 -datatype[26].structtype[0].compressthreshold 95 -datatype[26].structtype[0].compressminsize 800 -datatype[26].structtype[0].field[0].name "abyte" -datatype[26].structtype[0].field[0].datatype 16 -datatype[26].structtype[0].field[0].detailedtype "" -datatype[26].structtype[0].field[1].name "along" -datatype[26].structtype[0].field[1].datatype 4 -datatype[26].structtype[0].field[1].detailedtype "" -datatype[26].structtype[0].field[2].name "arrayfield" -datatype[26].structtype[0].field[2].datatype -1245117006 -datatype[26].structtype[0].field[2].detailedtype "" -datatype[26].structtype[0].field[3].name "setfield" -datatype[26].structtype[0].field[3].datatype 1328286588 -datatype[26].structtype[0].field[3].detailedtype "" -datatype[26].structtype[0].field[4].name "pos" -datatype[26].structtype[0].field[4].datatype 1381038251 -datatype[26].structtype[0].field[4].detailedtype "" -datatype[26].structtype[0].field[5].name "setfield2" -datatype[26].structtype[0].field[5].datatype 18 -datatype[26].structtype[0].field[5].detailedtype "" -datatype[26].structtype[0].field[6].name "setfield3" -datatype[26].structtype[0].field[6].datatype 2125328771 -datatype[26].structtype[0].field[6].detailedtype "" -datatype[26].structtype[0].field[7].name "setfield4" -datatype[26].structtype[0].field[7].datatype 2065577986 -datatype[26].structtype[0].field[7].detailedtype "" -datatype[26].structtype[0].field[8].name "tagfield" -datatype[26].structtype[0].field[8].datatype 18 -datatype[26].structtype[0].field[8].detailedtype "" -datatype[26].structtype[0].field[9].name "structfield" -datatype[26].structtype[0].field[9].datatype 109267174 -datatype[26].structtype[0].field[9].detailedtype "" -datatype[26].structtype[0].field[10].name "structarrayfield" -datatype[26].structtype[0].field[10].datatype -1244829667 -datatype[26].structtype[0].field[10].detailedtype "" -datatype[26].structtype[0].field[11].name "stringmapfield" -datatype[26].structtype[0].field[11].datatype 339965458 -datatype[26].structtype[0].field[11].detailedtype "" -datatype[26].structtype[0].field[12].name "intmapfield" -datatype[26].structtype[0].field[12].datatype -1584287606 -datatype[26].structtype[0].field[12].detailedtype "" -datatype[26].structtype[0].field[13].name "floatmapfield" -datatype[26].structtype[0].field[13].datatype 2125154557 -datatype[26].structtype[0].field[13].detailedtype "" -datatype[26].structtype[0].field[14].name "longmapfield" -datatype[26].structtype[0].field[14].datatype -1715531035 -datatype[26].structtype[0].field[14].detailedtype "" -datatype[26].structtype[0].field[15].name "doublemapfield" -datatype[26].structtype[0].field[15].datatype 2138385264 -datatype[26].structtype[0].field[15].detailedtype "" -datatype[26].structtype[0].field[16].name "arraymapfield" -datatype[26].structtype[0].field[16].datatype 435886609 -datatype[26].structtype[0].field[16].detailedtype "" -datatype[26].structtype[0].field[17].name "arrarr" -datatype[26].structtype[0].field[17].datatype -794985308 -datatype[26].structtype[0].field[17].detailedtype "" -datatype[26].structtype[0].field[18].name "maparr" -datatype[26].structtype[0].field[18].datatype 69621385 -datatype[26].structtype[0].field[18].detailedtype "" -datatype[26].structtype[0].field[19].name "complexarray" -datatype[26].structtype[0].field[19].datatype 1416345047 -datatype[26].structtype[0].field[19].detailedtype "" -datatype[26].structtype[0].field[20].name "mystructfield" -datatype[26].structtype[0].field[20].datatype -2092985853 -datatype[26].structtype[0].field[20].detailedtype "" -datatype[26].structtype[0].field[21].name "mystructmap" -datatype[26].structtype[0].field[21].datatype 1901258752 -datatype[26].structtype[0].field[21].detailedtype "" -datatype[26].structtype[0].field[22].name "mystructarr" -datatype[26].structtype[0].field[22].datatype 759956026 -datatype[26].structtype[0].field[22].detailedtype "" -datatype[26].structtype[0].field[23].name "Folders" -datatype[26].structtype[0].field[23].datatype -389833101 -datatype[26].structtype[0].field[23].detailedtype "" -datatype[26].structtype[0].field[24].name "juletre" -datatype[26].structtype[0].field[24].datatype 4 -datatype[26].structtype[0].field[24].detailedtype "" -datatype[26].structtype[0].field[25].name "album0" -datatype[26].structtype[0].field[25].datatype 18 -datatype[26].structtype[0].field[25].detailedtype "" -datatype[26].structtype[0].field[26].name "album1" -datatype[26].structtype[0].field[26].datatype 18 -datatype[26].structtype[0].field[26].detailedtype "" -datatype[26].structtype[0].field[27].name "other" -datatype[26].structtype[0].field[27].datatype 4 -datatype[26].structtype[0].field[27].detailedtype "" -datatype[27].id -853072901 -datatype[27].documenttype[0].name "types" -datatype[27].documenttype[0].version 0 -datatype[27].documenttype[0].inherits[0].name "document" -datatype[27].documenttype[0].inherits[0].version 0 -datatype[27].documenttype[0].headerstruct 1328581348 -datatype[27].documenttype[0].bodystruct 0 -datatype[27].documenttype[0].fieldsets{[document]}.fields[0] "Folders" -datatype[27].documenttype[0].fieldsets{[document]}.fields[1] "abyte" -datatype[27].documenttype[0].fieldsets{[document]}.fields[2] "album0" -datatype[27].documenttype[0].fieldsets{[document]}.fields[3] "album1" -datatype[27].documenttype[0].fieldsets{[document]}.fields[4] "along" -datatype[27].documenttype[0].fieldsets{[document]}.fields[5] "arrarr" -datatype[27].documenttype[0].fieldsets{[document]}.fields[6] "arrayfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[7] "arraymapfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[8] "complexarray" -datatype[27].documenttype[0].fieldsets{[document]}.fields[9] "doublemapfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[10] "floatmapfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[11] "intmapfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[12] "juletre" -datatype[27].documenttype[0].fieldsets{[document]}.fields[13] "longmapfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[14] "maparr" -datatype[27].documenttype[0].fieldsets{[document]}.fields[15] "mystructarr" -datatype[27].documenttype[0].fieldsets{[document]}.fields[16] "mystructfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[17] "mystructmap" -datatype[27].documenttype[0].fieldsets{[document]}.fields[18] "pos" -datatype[27].documenttype[0].fieldsets{[document]}.fields[19] "setfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[20] "setfield2" -datatype[27].documenttype[0].fieldsets{[document]}.fields[21] "setfield3" -datatype[27].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" -datatype[27].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[25] "structfield" -datatype[27].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "types" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "Folders" +doctype[1].fieldsets{[document]}.fields[1] "abyte" +doctype[1].fieldsets{[document]}.fields[2] "album0" +doctype[1].fieldsets{[document]}.fields[3] "album1" +doctype[1].fieldsets{[document]}.fields[4] "along" +doctype[1].fieldsets{[document]}.fields[5] "arrarr" +doctype[1].fieldsets{[document]}.fields[6] "arrayfield" +doctype[1].fieldsets{[document]}.fields[7] "arraymapfield" +doctype[1].fieldsets{[document]}.fields[8] "complexarray" +doctype[1].fieldsets{[document]}.fields[9] "doublemapfield" +doctype[1].fieldsets{[document]}.fields[10] "floatmapfield" +doctype[1].fieldsets{[document]}.fields[11] "intmapfield" +doctype[1].fieldsets{[document]}.fields[12] "juletre" +doctype[1].fieldsets{[document]}.fields[13] "longmapfield" +doctype[1].fieldsets{[document]}.fields[14] "maparr" +doctype[1].fieldsets{[document]}.fields[15] "mystructarr" +doctype[1].fieldsets{[document]}.fields[16] "mystructfield" +doctype[1].fieldsets{[document]}.fields[17] "mystructmap" +doctype[1].fieldsets{[document]}.fields[18] "pos" +doctype[1].fieldsets{[document]}.fields[19] "setfield" +doctype[1].fieldsets{[document]}.fields[20] "setfield2" +doctype[1].fieldsets{[document]}.fields[21] "setfield3" +doctype[1].fieldsets{[document]}.fields[22] "setfield4" +doctype[1].fieldsets{[document]}.fields[23] "stringmapfield" +doctype[1].fieldsets{[document]}.fields[24] "structarrayfield" +doctype[1].fieldsets{[document]}.fields[25] "structfield" +doctype[1].fieldsets{[document]}.fields[26] "tagfield" +doctype[1].arraytype[0].idx 10017 +doctype[1].arraytype[0].elementtype 10007 +doctype[1].arraytype[1].idx 10024 +doctype[1].arraytype[1].elementtype 10023 +doctype[1].arraytype[2].idx 10031 +doctype[1].arraytype[2].elementtype 10007 +doctype[1].arraytype[3].idx 10032 +doctype[1].arraytype[3].elementtype 10033 +doctype[1].arraytype[4].idx 10033 +doctype[1].arraytype[4].elementtype 10034 +doctype[1].arraytype[5].idx 10034 +doctype[1].arraytype[5].elementtype 10012 +doctype[1].arraytype[6].idx 10035 +doctype[1].arraytype[6].elementtype 10036 +doctype[1].arraytype[7].idx 10037 +doctype[1].arraytype[7].elementtype 10038 +doctype[1].arraytype[8].idx 10039 +doctype[1].arraytype[8].elementtype 10040 +doctype[1].arraytype[9].idx 10040 +doctype[1].arraytype[9].elementtype 10012 +doctype[1].arraytype[10].idx 10042 +doctype[1].arraytype[10].elementtype 10003 +doctype[1].arraytype[11].idx 10045 +doctype[1].arraytype[11].elementtype 10041 +doctype[1].maptype[0].idx 10025 +doctype[1].maptype[0].keytype 10012 +doctype[1].maptype[0].valuetype 10012 +doctype[1].maptype[1].idx 10026 +doctype[1].maptype[1].keytype 10012 +doctype[1].maptype[1].valuetype 10007 +doctype[1].maptype[2].idx 10027 +doctype[1].maptype[2].keytype 10012 +doctype[1].maptype[2].valuetype 10005 +doctype[1].maptype[3].idx 10028 +doctype[1].maptype[3].keytype 10007 +doctype[1].maptype[3].valuetype 10008 +doctype[1].maptype[4].idx 10029 +doctype[1].maptype[4].keytype 10007 +doctype[1].maptype[4].valuetype 10004 +doctype[1].maptype[5].idx 10030 +doctype[1].maptype[5].keytype 10012 +doctype[1].maptype[5].valuetype 10031 +doctype[1].maptype[6].idx 10036 +doctype[1].maptype[6].keytype 10012 +doctype[1].maptype[6].valuetype 10012 +doctype[1].maptype[7].idx 10038 +doctype[1].maptype[7].keytype 10007 +doctype[1].maptype[7].valuetype 10039 +doctype[1].maptype[8].idx 10043 +doctype[1].maptype[8].keytype 10012 +doctype[1].maptype[8].valuetype 10012 +doctype[1].maptype[9].idx 10044 +doctype[1].maptype[9].keytype 10007 +doctype[1].maptype[9].valuetype 10041 +doctype[1].maptype[10].idx 10046 +doctype[1].maptype[10].keytype 10007 +doctype[1].maptype[10].valuetype 10047 +doctype[1].maptype[11].idx 10048 +doctype[1].maptype[11].keytype 10012 +doctype[1].maptype[11].valuetype 10008 +doctype[1].wsettype[0].idx 10018 +doctype[1].wsettype[0].elementtype 10012 +doctype[1].wsettype[0].createifnonexistent false +doctype[1].wsettype[0].removeifzero false +doctype[1].wsettype[1].idx 10019 +doctype[1].wsettype[1].elementtype 10012 +doctype[1].wsettype[1].createifnonexistent true +doctype[1].wsettype[1].removeifzero true +doctype[1].wsettype[2].idx 10020 +doctype[1].wsettype[2].elementtype 10012 +doctype[1].wsettype[2].createifnonexistent false +doctype[1].wsettype[2].removeifzero true +doctype[1].wsettype[3].idx 10021 +doctype[1].wsettype[3].elementtype 10012 +doctype[1].wsettype[3].createifnonexistent true +doctype[1].wsettype[3].removeifzero false +doctype[1].wsettype[4].idx 10022 +doctype[1].wsettype[4].elementtype 10012 +doctype[1].wsettype[4].createifnonexistent true +doctype[1].wsettype[4].removeifzero true +doctype[1].wsettype[5].idx 10049 +doctype[1].wsettype[5].elementtype 10012 +doctype[1].wsettype[5].createifnonexistent true +doctype[1].wsettype[5].removeifzero true +doctype[1].wsettype[6].idx 10050 +doctype[1].wsettype[6].elementtype 10012 +doctype[1].wsettype[6].createifnonexistent true +doctype[1].wsettype[6].removeifzero true +doctype[1].structtype[0].idx 10023 +doctype[1].structtype[0].name "sct" +doctype[1].structtype[0].field[0].name "s1" +doctype[1].structtype[0].field[0].internalid 2146820765 +doctype[1].structtype[0].field[0].type 10012 +doctype[1].structtype[0].field[1].name "s2" +doctype[1].structtype[0].field[1].internalid 45366795 +doctype[1].structtype[0].field[1].type 10012 +doctype[1].structtype[1].idx 10041 +doctype[1].structtype[1].name "mystruct" +doctype[1].structtype[1].field[0].name "bytearr" +doctype[1].structtype[1].field[0].internalid 1079701754 +doctype[1].structtype[1].field[0].type 10042 +doctype[1].structtype[1].field[1].name "mymap" +doctype[1].structtype[1].field[1].internalid 1954178122 +doctype[1].structtype[1].field[1].type 10043 +doctype[1].structtype[1].field[2].name "title" +doctype[1].structtype[1].field[2].internalid 567626448 +doctype[1].structtype[1].field[2].type 10012 +doctype[1].structtype[1].field[3].name "structfield" +doctype[1].structtype[1].field[3].internalid 1726890940 +doctype[1].structtype[1].field[3].type 10012 +doctype[1].structtype[2].idx 10047 +doctype[1].structtype[2].name "folder" +doctype[1].structtype[2].field[0].name "Version" +doctype[1].structtype[2].field[0].internalid 64430502 +doctype[1].structtype[2].field[0].type 10007 +doctype[1].structtype[2].field[1].name "Name" +doctype[1].structtype[2].field[1].internalid 2002760220 +doctype[1].structtype[2].field[1].type 10012 +doctype[1].structtype[2].field[2].name "FlagsCounter" +doctype[1].structtype[2].field[2].internalid 1741227606 +doctype[1].structtype[2].field[2].type 10048 +doctype[1].structtype[2].field[3].name "anotherfolder" +doctype[1].structtype[2].field[3].internalid 1582421848 +doctype[1].structtype[2].field[3].type 10047 +doctype[1].structtype[3].idx 10016 +doctype[1].structtype[3].name "types.header" +doctype[1].structtype[3].field[0].name "abyte" +doctype[1].structtype[3].field[0].internalid 110138156 +doctype[1].structtype[3].field[0].type 10003 +doctype[1].structtype[3].field[1].name "along" +doctype[1].structtype[3].field[1].internalid 1206464520 +doctype[1].structtype[3].field[1].type 10008 +doctype[1].structtype[3].field[2].name "arrayfield" +doctype[1].structtype[3].field[2].internalid 965790107 +doctype[1].structtype[3].field[2].type 10017 +doctype[1].structtype[3].field[3].name "setfield" +doctype[1].structtype[3].field[3].internalid 761581914 +doctype[1].structtype[3].field[3].type 10018 +doctype[1].structtype[3].field[4].name "pos" +doctype[1].structtype[3].field[4].internalid 1041567475 +doctype[1].structtype[3].field[4].type 10009 +doctype[1].structtype[3].field[5].name "setfield2" +doctype[1].structtype[3].field[5].internalid 1066659198 +doctype[1].structtype[3].field[5].type 10019 +doctype[1].structtype[3].field[6].name "setfield3" +doctype[1].structtype[3].field[6].internalid 1180155772 +doctype[1].structtype[3].field[6].type 10020 +doctype[1].structtype[3].field[7].name "setfield4" +doctype[1].structtype[3].field[7].internalid 1254131631 +doctype[1].structtype[3].field[7].type 10021 +doctype[1].structtype[3].field[8].name "tagfield" +doctype[1].structtype[3].field[8].internalid 1653562069 +doctype[1].structtype[3].field[8].type 10022 +doctype[1].structtype[3].field[9].name "structfield" +doctype[1].structtype[3].field[9].internalid 486207386 +doctype[1].structtype[3].field[9].type 10023 +doctype[1].structtype[3].field[10].name "structarrayfield" +doctype[1].structtype[3].field[10].internalid 335048518 +doctype[1].structtype[3].field[10].type 10024 +doctype[1].structtype[3].field[11].name "stringmapfield" +doctype[1].structtype[3].field[11].internalid 117465687 +doctype[1].structtype[3].field[11].type 10025 +doctype[1].structtype[3].field[12].name "intmapfield" +doctype[1].structtype[3].field[12].internalid 121004462 +doctype[1].structtype[3].field[12].type 10026 +doctype[1].structtype[3].field[13].name "floatmapfield" +doctype[1].structtype[3].field[13].internalid 1239120925 +doctype[1].structtype[3].field[13].type 10027 +doctype[1].structtype[3].field[14].name "longmapfield" +doctype[1].structtype[3].field[14].internalid 477718745 +doctype[1].structtype[3].field[14].type 10028 +doctype[1].structtype[3].field[15].name "doublemapfield" +doctype[1].structtype[3].field[15].internalid 877047192 +doctype[1].structtype[3].field[15].type 10029 +doctype[1].structtype[3].field[16].name "arraymapfield" +doctype[1].structtype[3].field[16].internalid 1670805928 +doctype[1].structtype[3].field[16].type 10030 +doctype[1].structtype[3].field[17].name "arrarr" +doctype[1].structtype[3].field[17].internalid 1962567166 +doctype[1].structtype[3].field[17].type 10032 +doctype[1].structtype[3].field[18].name "maparr" +doctype[1].structtype[3].field[18].internalid 904375219 +doctype[1].structtype[3].field[18].type 10035 +doctype[1].structtype[3].field[19].name "complexarray" +doctype[1].structtype[3].field[19].internalid 795629533 +doctype[1].structtype[3].field[19].type 10037 +doctype[1].structtype[3].field[20].name "mystructfield" +doctype[1].structtype[3].field[20].internalid 1348513378 +doctype[1].structtype[3].field[20].type 10041 +doctype[1].structtype[3].field[21].name "mystructmap" +doctype[1].structtype[3].field[21].internalid 1511423250 +doctype[1].structtype[3].field[21].type 10044 +doctype[1].structtype[3].field[22].name "mystructarr" +doctype[1].structtype[3].field[22].internalid 595856991 +doctype[1].structtype[3].field[22].type 10045 +doctype[1].structtype[3].field[23].name "Folders" +doctype[1].structtype[3].field[23].internalid 34575524 +doctype[1].structtype[3].field[23].type 10046 +doctype[1].structtype[3].field[24].name "juletre" +doctype[1].structtype[3].field[24].internalid 1039981530 +doctype[1].structtype[3].field[24].type 10008 +doctype[1].structtype[3].field[25].name "album0" +doctype[1].structtype[3].field[25].internalid 764312262 +doctype[1].structtype[3].field[25].type 10049 +doctype[1].structtype[3].field[26].name "album1" +doctype[1].structtype[3].field[26].internalid 1967160809 +doctype[1].structtype[3].field[26].type 10050 +doctype[1].structtype[3].field[27].name "other" +doctype[1].structtype[3].field[27].internalid 2443357 +doctype[1].structtype[3].field[27].type 10008 diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index 94551567352..0501aa58784 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -6,11 +6,11 @@ documenttype[0].version 0 documenttype[0].headerstruct 1328581348 documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id -1865479609 -documenttype[0].datatype[0].type MAP -documenttype[0].datatype[0].array.element.id 0 -documenttype[0].datatype[0].map.key.id 2 -documenttype[0].datatype[0].map.value.id 4 +documenttype[0].datatype[0].id -1486737430 +documenttype[0].datatype[0].type ARRAY +documenttype[0].datatype[0].array.element.id 2 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 documenttype[0].datatype[0].wset.key.id 0 documenttype[0].datatype[0].wset.createifnonexistent false documenttype[0].datatype[0].wset.removeifzero false @@ -21,65 +21,41 @@ documenttype[0].datatype[0].sstruct.compression.type NONE documenttype[0].datatype[0].sstruct.compression.level 0 documenttype[0].datatype[0].sstruct.compression.threshold 95 documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[1].id 294108848 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].id 1707615575 +documenttype[0].datatype[1].type ARRAY +documenttype[0].datatype[1].array.element.id -1486737430 documenttype[0].datatype[1].map.key.id 0 documenttype[0].datatype[1].map.value.id 0 documenttype[0].datatype[1].wset.key.id 0 documenttype[0].datatype[1].wset.createifnonexistent false documenttype[0].datatype[1].wset.removeifzero false documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "folder" +documenttype[0].datatype[1].sstruct.name "" documenttype[0].datatype[1].sstruct.version 0 documenttype[0].datatype[1].sstruct.compression.type NONE documenttype[0].datatype[1].sstruct.compression.level 0 documenttype[0].datatype[1].sstruct.compression.threshold 95 documenttype[0].datatype[1].sstruct.compression.minsize 200 -documenttype[0].datatype[1].sstruct.field[0].name "Version" -documenttype[0].datatype[1].sstruct.field[0].id 64430502 -documenttype[0].datatype[1].sstruct.field[0].datatype 0 -documenttype[0].datatype[1].sstruct.field[0].detailedtype "" -documenttype[0].datatype[1].sstruct.field[1].name "Name" -documenttype[0].datatype[1].sstruct.field[1].id 2002760220 -documenttype[0].datatype[1].sstruct.field[1].datatype 2 -documenttype[0].datatype[1].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].sstruct.field[2].name "FlagsCounter" -documenttype[0].datatype[1].sstruct.field[2].id 1741227606 -documenttype[0].datatype[1].sstruct.field[2].datatype -1865479609 -documenttype[0].datatype[1].sstruct.field[2].detailedtype "" -documenttype[0].datatype[1].sstruct.field[3].name "anotherfolder" -documenttype[0].datatype[1].sstruct.field[3].id 1582421848 -documenttype[0].datatype[1].sstruct.field[3].datatype 294108848 -documenttype[0].datatype[1].sstruct.field[3].detailedtype "" -documenttype[0].datatype[2].id 109267174 -documenttype[0].datatype[2].type STRUCT -documenttype[0].datatype[2].array.element.id 0 +documenttype[0].datatype[2].id -794985308 +documenttype[0].datatype[2].type ARRAY +documenttype[0].datatype[2].array.element.id 1707615575 documenttype[0].datatype[2].map.key.id 0 documenttype[0].datatype[2].map.value.id 0 documenttype[0].datatype[2].wset.key.id 0 documenttype[0].datatype[2].wset.createifnonexistent false documenttype[0].datatype[2].wset.removeifzero false documenttype[0].datatype[2].annotationref.annotation.id 0 -documenttype[0].datatype[2].sstruct.name "sct" +documenttype[0].datatype[2].sstruct.name "" documenttype[0].datatype[2].sstruct.version 0 documenttype[0].datatype[2].sstruct.compression.type NONE documenttype[0].datatype[2].sstruct.compression.level 0 documenttype[0].datatype[2].sstruct.compression.threshold 95 documenttype[0].datatype[2].sstruct.compression.minsize 200 -documenttype[0].datatype[2].sstruct.field[0].name "s1" -documenttype[0].datatype[2].sstruct.field[0].id 2146820765 -documenttype[0].datatype[2].sstruct.field[0].datatype 2 -documenttype[0].datatype[2].sstruct.field[0].detailedtype "" -documenttype[0].datatype[2].sstruct.field[1].name "s2" -documenttype[0].datatype[2].sstruct.field[1].id 45366795 -documenttype[0].datatype[2].sstruct.field[1].datatype 2 -documenttype[0].datatype[2].sstruct.field[1].detailedtype "" -documenttype[0].datatype[3].id 49942803 -documenttype[0].datatype[3].type ARRAY -documenttype[0].datatype[3].array.element.id 16 +documenttype[0].datatype[3].id -372512406 +documenttype[0].datatype[3].type MAP +documenttype[0].datatype[3].array.element.id 0 documenttype[0].datatype[3].map.key.id 0 -documenttype[0].datatype[3].map.value.id 0 +documenttype[0].datatype[3].map.value.id 1707615575 documenttype[0].datatype[3].wset.key.id 0 documenttype[0].datatype[3].wset.createifnonexistent false documenttype[0].datatype[3].wset.removeifzero false @@ -90,11 +66,11 @@ documenttype[0].datatype[3].sstruct.compression.type NONE documenttype[0].datatype[3].sstruct.compression.level 0 documenttype[0].datatype[3].sstruct.compression.threshold 95 documenttype[0].datatype[3].sstruct.compression.minsize 200 -documenttype[0].datatype[4].id 339965458 -documenttype[0].datatype[4].type MAP -documenttype[0].datatype[4].array.element.id 0 -documenttype[0].datatype[4].map.key.id 2 -documenttype[0].datatype[4].map.value.id 2 +documenttype[0].datatype[4].id 1416345047 +documenttype[0].datatype[4].type ARRAY +documenttype[0].datatype[4].array.element.id -372512406 +documenttype[0].datatype[4].map.key.id 0 +documenttype[0].datatype[4].map.value.id 0 documenttype[0].datatype[4].wset.key.id 0 documenttype[0].datatype[4].wset.createifnonexistent false documenttype[0].datatype[4].wset.removeifzero false @@ -105,40 +81,24 @@ documenttype[0].datatype[4].sstruct.compression.type NONE documenttype[0].datatype[4].sstruct.compression.level 0 documenttype[0].datatype[4].sstruct.compression.threshold 95 documenttype[0].datatype[4].sstruct.compression.minsize 200 -documenttype[0].datatype[5].id -2092985853 -documenttype[0].datatype[5].type STRUCT +documenttype[0].datatype[5].id 339965458 +documenttype[0].datatype[5].type MAP documenttype[0].datatype[5].array.element.id 0 -documenttype[0].datatype[5].map.key.id 0 -documenttype[0].datatype[5].map.value.id 0 +documenttype[0].datatype[5].map.key.id 2 +documenttype[0].datatype[5].map.value.id 2 documenttype[0].datatype[5].wset.key.id 0 documenttype[0].datatype[5].wset.createifnonexistent false documenttype[0].datatype[5].wset.removeifzero false documenttype[0].datatype[5].annotationref.annotation.id 0 -documenttype[0].datatype[5].sstruct.name "mystruct" +documenttype[0].datatype[5].sstruct.name "" documenttype[0].datatype[5].sstruct.version 0 documenttype[0].datatype[5].sstruct.compression.type NONE documenttype[0].datatype[5].sstruct.compression.level 0 documenttype[0].datatype[5].sstruct.compression.threshold 95 documenttype[0].datatype[5].sstruct.compression.minsize 200 -documenttype[0].datatype[5].sstruct.field[0].name "bytearr" -documenttype[0].datatype[5].sstruct.field[0].id 1079701754 -documenttype[0].datatype[5].sstruct.field[0].datatype 49942803 -documenttype[0].datatype[5].sstruct.field[0].detailedtype "" -documenttype[0].datatype[5].sstruct.field[1].name "mymap" -documenttype[0].datatype[5].sstruct.field[1].id 1954178122 -documenttype[0].datatype[5].sstruct.field[1].datatype 339965458 -documenttype[0].datatype[5].sstruct.field[1].detailedtype "" -documenttype[0].datatype[5].sstruct.field[2].name "title" -documenttype[0].datatype[5].sstruct.field[2].id 567626448 -documenttype[0].datatype[5].sstruct.field[2].datatype 2 -documenttype[0].datatype[5].sstruct.field[2].detailedtype "" -documenttype[0].datatype[5].sstruct.field[3].name "structfield" -documenttype[0].datatype[5].sstruct.field[3].id 1726890940 -documenttype[0].datatype[5].sstruct.field[3].datatype 2 -documenttype[0].datatype[5].sstruct.field[3].detailedtype "" -documenttype[0].datatype[6].id -1245117006 +documenttype[0].datatype[6].id 69621385 documenttype[0].datatype[6].type ARRAY -documenttype[0].datatype[6].array.element.id 0 +documenttype[0].datatype[6].array.element.id 339965458 documenttype[0].datatype[6].map.key.id 0 documenttype[0].datatype[6].map.value.id 0 documenttype[0].datatype[6].wset.key.id 0 @@ -151,12 +111,12 @@ documenttype[0].datatype[6].sstruct.compression.type NONE documenttype[0].datatype[6].sstruct.compression.level 0 documenttype[0].datatype[6].sstruct.compression.threshold 95 documenttype[0].datatype[6].sstruct.compression.minsize 200 -documenttype[0].datatype[7].id 1328286588 -documenttype[0].datatype[7].type WSET -documenttype[0].datatype[7].array.element.id 0 +documenttype[0].datatype[7].id 49942803 +documenttype[0].datatype[7].type ARRAY +documenttype[0].datatype[7].array.element.id 16 documenttype[0].datatype[7].map.key.id 0 documenttype[0].datatype[7].map.value.id 0 -documenttype[0].datatype[7].wset.key.id 2 +documenttype[0].datatype[7].wset.key.id 0 documenttype[0].datatype[7].wset.createifnonexistent false documenttype[0].datatype[7].wset.removeifzero false documenttype[0].datatype[7].annotationref.annotation.id 0 @@ -166,14 +126,14 @@ documenttype[0].datatype[7].sstruct.compression.type NONE documenttype[0].datatype[7].sstruct.compression.level 0 documenttype[0].datatype[7].sstruct.compression.threshold 95 documenttype[0].datatype[7].sstruct.compression.minsize 200 -documenttype[0].datatype[8].id 2125328771 -documenttype[0].datatype[8].type WSET +documenttype[0].datatype[8].id -1245117006 +documenttype[0].datatype[8].type ARRAY documenttype[0].datatype[8].array.element.id 0 documenttype[0].datatype[8].map.key.id 0 documenttype[0].datatype[8].map.value.id 0 -documenttype[0].datatype[8].wset.key.id 2 +documenttype[0].datatype[8].wset.key.id 0 documenttype[0].datatype[8].wset.createifnonexistent false -documenttype[0].datatype[8].wset.removeifzero true +documenttype[0].datatype[8].wset.removeifzero false documenttype[0].datatype[8].annotationref.annotation.id 0 documenttype[0].datatype[8].sstruct.name "" documenttype[0].datatype[8].sstruct.version 0 @@ -181,24 +141,40 @@ documenttype[0].datatype[8].sstruct.compression.type NONE documenttype[0].datatype[8].sstruct.compression.level 0 documenttype[0].datatype[8].sstruct.compression.threshold 95 documenttype[0].datatype[8].sstruct.compression.minsize 200 -documenttype[0].datatype[9].id 2065577986 -documenttype[0].datatype[9].type WSET +documenttype[0].datatype[9].id -2092985853 +documenttype[0].datatype[9].type STRUCT documenttype[0].datatype[9].array.element.id 0 documenttype[0].datatype[9].map.key.id 0 documenttype[0].datatype[9].map.value.id 0 -documenttype[0].datatype[9].wset.key.id 2 -documenttype[0].datatype[9].wset.createifnonexistent true +documenttype[0].datatype[9].wset.key.id 0 +documenttype[0].datatype[9].wset.createifnonexistent false documenttype[0].datatype[9].wset.removeifzero false documenttype[0].datatype[9].annotationref.annotation.id 0 -documenttype[0].datatype[9].sstruct.name "" +documenttype[0].datatype[9].sstruct.name "mystruct" documenttype[0].datatype[9].sstruct.version 0 documenttype[0].datatype[9].sstruct.compression.type NONE documenttype[0].datatype[9].sstruct.compression.level 0 documenttype[0].datatype[9].sstruct.compression.threshold 95 documenttype[0].datatype[9].sstruct.compression.minsize 200 -documenttype[0].datatype[10].id -1244829667 +documenttype[0].datatype[9].sstruct.field[0].name "bytearr" +documenttype[0].datatype[9].sstruct.field[0].id 1079701754 +documenttype[0].datatype[9].sstruct.field[0].datatype 49942803 +documenttype[0].datatype[9].sstruct.field[0].detailedtype "" +documenttype[0].datatype[9].sstruct.field[1].name "mymap" +documenttype[0].datatype[9].sstruct.field[1].id 1954178122 +documenttype[0].datatype[9].sstruct.field[1].datatype 339965458 +documenttype[0].datatype[9].sstruct.field[1].detailedtype "" +documenttype[0].datatype[9].sstruct.field[2].name "title" +documenttype[0].datatype[9].sstruct.field[2].id 567626448 +documenttype[0].datatype[9].sstruct.field[2].datatype 2 +documenttype[0].datatype[9].sstruct.field[2].detailedtype "" +documenttype[0].datatype[9].sstruct.field[3].name "structfield" +documenttype[0].datatype[9].sstruct.field[3].id 1726890940 +documenttype[0].datatype[9].sstruct.field[3].datatype 2 +documenttype[0].datatype[9].sstruct.field[3].detailedtype "" +documenttype[0].datatype[10].id 759956026 documenttype[0].datatype[10].type ARRAY -documenttype[0].datatype[10].array.element.id 109267174 +documenttype[0].datatype[10].array.element.id -2092985853 documenttype[0].datatype[10].map.key.id 0 documenttype[0].datatype[10].map.value.id 0 documenttype[0].datatype[10].wset.key.id 0 @@ -211,26 +187,34 @@ documenttype[0].datatype[10].sstruct.compression.type NONE documenttype[0].datatype[10].sstruct.compression.level 0 documenttype[0].datatype[10].sstruct.compression.threshold 95 documenttype[0].datatype[10].sstruct.compression.minsize 200 -documenttype[0].datatype[11].id -1584287606 -documenttype[0].datatype[11].type MAP +documenttype[0].datatype[11].id 109267174 +documenttype[0].datatype[11].type STRUCT documenttype[0].datatype[11].array.element.id 0 -documenttype[0].datatype[11].map.key.id 2 +documenttype[0].datatype[11].map.key.id 0 documenttype[0].datatype[11].map.value.id 0 documenttype[0].datatype[11].wset.key.id 0 documenttype[0].datatype[11].wset.createifnonexistent false documenttype[0].datatype[11].wset.removeifzero false documenttype[0].datatype[11].annotationref.annotation.id 0 -documenttype[0].datatype[11].sstruct.name "" +documenttype[0].datatype[11].sstruct.name "sct" documenttype[0].datatype[11].sstruct.version 0 documenttype[0].datatype[11].sstruct.compression.type NONE documenttype[0].datatype[11].sstruct.compression.level 0 documenttype[0].datatype[11].sstruct.compression.threshold 95 documenttype[0].datatype[11].sstruct.compression.minsize 200 -documenttype[0].datatype[12].id 2125154557 -documenttype[0].datatype[12].type MAP -documenttype[0].datatype[12].array.element.id 0 -documenttype[0].datatype[12].map.key.id 2 -documenttype[0].datatype[12].map.value.id 1 +documenttype[0].datatype[11].sstruct.field[0].name "s1" +documenttype[0].datatype[11].sstruct.field[0].id 2146820765 +documenttype[0].datatype[11].sstruct.field[0].datatype 2 +documenttype[0].datatype[11].sstruct.field[0].detailedtype "" +documenttype[0].datatype[11].sstruct.field[1].name "s2" +documenttype[0].datatype[11].sstruct.field[1].id 45366795 +documenttype[0].datatype[11].sstruct.field[1].datatype 2 +documenttype[0].datatype[11].sstruct.field[1].detailedtype "" +documenttype[0].datatype[12].id -1244829667 +documenttype[0].datatype[12].type ARRAY +documenttype[0].datatype[12].array.element.id 109267174 +documenttype[0].datatype[12].map.key.id 0 +documenttype[0].datatype[12].map.value.id 0 documenttype[0].datatype[12].wset.key.id 0 documenttype[0].datatype[12].wset.createifnonexistent false documenttype[0].datatype[12].wset.removeifzero false @@ -241,11 +225,11 @@ documenttype[0].datatype[12].sstruct.compression.type NONE documenttype[0].datatype[12].sstruct.compression.level 0 documenttype[0].datatype[12].sstruct.compression.threshold 95 documenttype[0].datatype[12].sstruct.compression.minsize 200 -documenttype[0].datatype[13].id -1715531035 +documenttype[0].datatype[13].id 2138385264 documenttype[0].datatype[13].type MAP documenttype[0].datatype[13].array.element.id 0 documenttype[0].datatype[13].map.key.id 0 -documenttype[0].datatype[13].map.value.id 4 +documenttype[0].datatype[13].map.value.id 5 documenttype[0].datatype[13].wset.key.id 0 documenttype[0].datatype[13].wset.createifnonexistent false documenttype[0].datatype[13].wset.removeifzero false @@ -256,11 +240,11 @@ documenttype[0].datatype[13].sstruct.compression.type NONE documenttype[0].datatype[13].sstruct.compression.level 0 documenttype[0].datatype[13].sstruct.compression.threshold 95 documenttype[0].datatype[13].sstruct.compression.minsize 200 -documenttype[0].datatype[14].id 2138385264 +documenttype[0].datatype[14].id -1865479609 documenttype[0].datatype[14].type MAP documenttype[0].datatype[14].array.element.id 0 -documenttype[0].datatype[14].map.key.id 0 -documenttype[0].datatype[14].map.value.id 5 +documenttype[0].datatype[14].map.key.id 2 +documenttype[0].datatype[14].map.value.id 4 documenttype[0].datatype[14].wset.key.id 0 documenttype[0].datatype[14].wset.createifnonexistent false documenttype[0].datatype[14].wset.removeifzero false @@ -271,26 +255,42 @@ documenttype[0].datatype[14].sstruct.compression.type NONE documenttype[0].datatype[14].sstruct.compression.level 0 documenttype[0].datatype[14].sstruct.compression.threshold 95 documenttype[0].datatype[14].sstruct.compression.minsize 200 -documenttype[0].datatype[15].id 435886609 -documenttype[0].datatype[15].type MAP +documenttype[0].datatype[15].id 294108848 +documenttype[0].datatype[15].type STRUCT documenttype[0].datatype[15].array.element.id 0 -documenttype[0].datatype[15].map.key.id 2 -documenttype[0].datatype[15].map.value.id -1245117006 +documenttype[0].datatype[15].map.key.id 0 +documenttype[0].datatype[15].map.value.id 0 documenttype[0].datatype[15].wset.key.id 0 documenttype[0].datatype[15].wset.createifnonexistent false documenttype[0].datatype[15].wset.removeifzero false documenttype[0].datatype[15].annotationref.annotation.id 0 -documenttype[0].datatype[15].sstruct.name "" +documenttype[0].datatype[15].sstruct.name "folder" documenttype[0].datatype[15].sstruct.version 0 documenttype[0].datatype[15].sstruct.compression.type NONE documenttype[0].datatype[15].sstruct.compression.level 0 documenttype[0].datatype[15].sstruct.compression.threshold 95 documenttype[0].datatype[15].sstruct.compression.minsize 200 -documenttype[0].datatype[16].id -1486737430 -documenttype[0].datatype[16].type ARRAY -documenttype[0].datatype[16].array.element.id 2 +documenttype[0].datatype[15].sstruct.field[0].name "Version" +documenttype[0].datatype[15].sstruct.field[0].id 64430502 +documenttype[0].datatype[15].sstruct.field[0].datatype 0 +documenttype[0].datatype[15].sstruct.field[0].detailedtype "" +documenttype[0].datatype[15].sstruct.field[1].name "Name" +documenttype[0].datatype[15].sstruct.field[1].id 2002760220 +documenttype[0].datatype[15].sstruct.field[1].datatype 2 +documenttype[0].datatype[15].sstruct.field[1].detailedtype "" +documenttype[0].datatype[15].sstruct.field[2].name "FlagsCounter" +documenttype[0].datatype[15].sstruct.field[2].id 1741227606 +documenttype[0].datatype[15].sstruct.field[2].datatype -1865479609 +documenttype[0].datatype[15].sstruct.field[2].detailedtype "" +documenttype[0].datatype[15].sstruct.field[3].name "anotherfolder" +documenttype[0].datatype[15].sstruct.field[3].id 1582421848 +documenttype[0].datatype[15].sstruct.field[3].datatype 294108848 +documenttype[0].datatype[15].sstruct.field[3].detailedtype "" +documenttype[0].datatype[16].id -389833101 +documenttype[0].datatype[16].type MAP +documenttype[0].datatype[16].array.element.id 0 documenttype[0].datatype[16].map.key.id 0 -documenttype[0].datatype[16].map.value.id 0 +documenttype[0].datatype[16].map.value.id 294108848 documenttype[0].datatype[16].wset.key.id 0 documenttype[0].datatype[16].wset.createifnonexistent false documenttype[0].datatype[16].wset.removeifzero false @@ -301,11 +301,11 @@ documenttype[0].datatype[16].sstruct.compression.type NONE documenttype[0].datatype[16].sstruct.compression.level 0 documenttype[0].datatype[16].sstruct.compression.threshold 95 documenttype[0].datatype[16].sstruct.compression.minsize 200 -documenttype[0].datatype[17].id 1707615575 -documenttype[0].datatype[17].type ARRAY -documenttype[0].datatype[17].array.element.id -1486737430 +documenttype[0].datatype[17].id -1715531035 +documenttype[0].datatype[17].type MAP +documenttype[0].datatype[17].array.element.id 0 documenttype[0].datatype[17].map.key.id 0 -documenttype[0].datatype[17].map.value.id 0 +documenttype[0].datatype[17].map.value.id 4 documenttype[0].datatype[17].wset.key.id 0 documenttype[0].datatype[17].wset.createifnonexistent false documenttype[0].datatype[17].wset.removeifzero false @@ -316,11 +316,11 @@ documenttype[0].datatype[17].sstruct.compression.type NONE documenttype[0].datatype[17].sstruct.compression.level 0 documenttype[0].datatype[17].sstruct.compression.threshold 95 documenttype[0].datatype[17].sstruct.compression.minsize 200 -documenttype[0].datatype[18].id -794985308 -documenttype[0].datatype[18].type ARRAY -documenttype[0].datatype[18].array.element.id 1707615575 +documenttype[0].datatype[18].id 1901258752 +documenttype[0].datatype[18].type MAP +documenttype[0].datatype[18].array.element.id 0 documenttype[0].datatype[18].map.key.id 0 -documenttype[0].datatype[18].map.value.id 0 +documenttype[0].datatype[18].map.value.id -2092985853 documenttype[0].datatype[18].wset.key.id 0 documenttype[0].datatype[18].wset.createifnonexistent false documenttype[0].datatype[18].wset.removeifzero false @@ -331,11 +331,11 @@ documenttype[0].datatype[18].sstruct.compression.type NONE documenttype[0].datatype[18].sstruct.compression.level 0 documenttype[0].datatype[18].sstruct.compression.threshold 95 documenttype[0].datatype[18].sstruct.compression.minsize 200 -documenttype[0].datatype[19].id 69621385 -documenttype[0].datatype[19].type ARRAY -documenttype[0].datatype[19].array.element.id 339965458 -documenttype[0].datatype[19].map.key.id 0 -documenttype[0].datatype[19].map.value.id 0 +documenttype[0].datatype[19].id 435886609 +documenttype[0].datatype[19].type MAP +documenttype[0].datatype[19].array.element.id 0 +documenttype[0].datatype[19].map.key.id 2 +documenttype[0].datatype[19].map.value.id -1245117006 documenttype[0].datatype[19].wset.key.id 0 documenttype[0].datatype[19].wset.createifnonexistent false documenttype[0].datatype[19].wset.removeifzero false @@ -346,11 +346,11 @@ documenttype[0].datatype[19].sstruct.compression.type NONE documenttype[0].datatype[19].sstruct.compression.level 0 documenttype[0].datatype[19].sstruct.compression.threshold 95 documenttype[0].datatype[19].sstruct.compression.minsize 200 -documenttype[0].datatype[20].id -372512406 +documenttype[0].datatype[20].id 2125154557 documenttype[0].datatype[20].type MAP documenttype[0].datatype[20].array.element.id 0 -documenttype[0].datatype[20].map.key.id 0 -documenttype[0].datatype[20].map.value.id 1707615575 +documenttype[0].datatype[20].map.key.id 2 +documenttype[0].datatype[20].map.value.id 1 documenttype[0].datatype[20].wset.key.id 0 documenttype[0].datatype[20].wset.createifnonexistent false documenttype[0].datatype[20].wset.removeifzero false @@ -361,10 +361,10 @@ documenttype[0].datatype[20].sstruct.compression.type NONE documenttype[0].datatype[20].sstruct.compression.level 0 documenttype[0].datatype[20].sstruct.compression.threshold 95 documenttype[0].datatype[20].sstruct.compression.minsize 200 -documenttype[0].datatype[21].id 1416345047 -documenttype[0].datatype[21].type ARRAY -documenttype[0].datatype[21].array.element.id -372512406 -documenttype[0].datatype[21].map.key.id 0 +documenttype[0].datatype[21].id -1584287606 +documenttype[0].datatype[21].type MAP +documenttype[0].datatype[21].array.element.id 0 +documenttype[0].datatype[21].map.key.id 2 documenttype[0].datatype[21].map.value.id 0 documenttype[0].datatype[21].wset.key.id 0 documenttype[0].datatype[21].wset.createifnonexistent false @@ -376,12 +376,12 @@ documenttype[0].datatype[21].sstruct.compression.type NONE documenttype[0].datatype[21].sstruct.compression.level 0 documenttype[0].datatype[21].sstruct.compression.threshold 95 documenttype[0].datatype[21].sstruct.compression.minsize 200 -documenttype[0].datatype[22].id 1901258752 -documenttype[0].datatype[22].type MAP +documenttype[0].datatype[22].id 1328286588 +documenttype[0].datatype[22].type WSET documenttype[0].datatype[22].array.element.id 0 documenttype[0].datatype[22].map.key.id 0 -documenttype[0].datatype[22].map.value.id -2092985853 -documenttype[0].datatype[22].wset.key.id 0 +documenttype[0].datatype[22].map.value.id 0 +documenttype[0].datatype[22].wset.key.id 2 documenttype[0].datatype[22].wset.createifnonexistent false documenttype[0].datatype[22].wset.removeifzero false documenttype[0].datatype[22].annotationref.annotation.id 0 @@ -391,13 +391,13 @@ documenttype[0].datatype[22].sstruct.compression.type NONE documenttype[0].datatype[22].sstruct.compression.level 0 documenttype[0].datatype[22].sstruct.compression.threshold 95 documenttype[0].datatype[22].sstruct.compression.minsize 200 -documenttype[0].datatype[23].id 759956026 -documenttype[0].datatype[23].type ARRAY -documenttype[0].datatype[23].array.element.id -2092985853 +documenttype[0].datatype[23].id 2065577986 +documenttype[0].datatype[23].type WSET +documenttype[0].datatype[23].array.element.id 0 documenttype[0].datatype[23].map.key.id 0 documenttype[0].datatype[23].map.value.id 0 -documenttype[0].datatype[23].wset.key.id 0 -documenttype[0].datatype[23].wset.createifnonexistent false +documenttype[0].datatype[23].wset.key.id 2 +documenttype[0].datatype[23].wset.createifnonexistent true documenttype[0].datatype[23].wset.removeifzero false documenttype[0].datatype[23].annotationref.annotation.id 0 documenttype[0].datatype[23].sstruct.name "" @@ -406,14 +406,14 @@ documenttype[0].datatype[23].sstruct.compression.type NONE documenttype[0].datatype[23].sstruct.compression.level 0 documenttype[0].datatype[23].sstruct.compression.threshold 95 documenttype[0].datatype[23].sstruct.compression.minsize 200 -documenttype[0].datatype[24].id -389833101 -documenttype[0].datatype[24].type MAP +documenttype[0].datatype[24].id 2125328771 +documenttype[0].datatype[24].type WSET documenttype[0].datatype[24].array.element.id 0 documenttype[0].datatype[24].map.key.id 0 -documenttype[0].datatype[24].map.value.id 294108848 -documenttype[0].datatype[24].wset.key.id 0 +documenttype[0].datatype[24].map.value.id 0 +documenttype[0].datatype[24].wset.key.id 2 documenttype[0].datatype[24].wset.createifnonexistent false -documenttype[0].datatype[24].wset.removeifzero false +documenttype[0].datatype[24].wset.removeifzero true documenttype[0].datatype[24].annotationref.annotation.id 0 documenttype[0].datatype[24].sstruct.name "" documenttype[0].datatype[24].sstruct.version 0 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index bf7632a504c..eaa7e2c5131 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -1,84 +1,86 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id 595216861 -datatype[1].referencetype[0].target_type_id -1318255918 -datatype[2].id 542332920 -datatype[2].referencetype[0].target_type_id 443162583 -datatype[3].id 959075962 -datatype[3].structtype[0].name "ad.header" -datatype[3].structtype[0].version 0 -datatype[3].structtype[0].compresstype NONE -datatype[3].structtype[0].compresslevel 0 -datatype[3].structtype[0].compressthreshold 95 -datatype[3].structtype[0].compressminsize 800 -datatype[3].structtype[0].field[0].name "campaign_ref" -datatype[3].structtype[0].field[0].datatype 595216861 -datatype[3].structtype[0].field[0].detailedtype "" -datatype[3].structtype[0].field[1].name "person_ref" -datatype[3].structtype[0].field[1].datatype 542332920 -datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id 2987301 -datatype[4].documenttype[0].name "ad" -datatype[4].documenttype[0].version 0 -datatype[4].documenttype[0].inherits[0].name "document" -datatype[4].documenttype[0].inherits[0].version 0 -datatype[4].documenttype[0].headerstruct 959075962 -datatype[4].documenttype[0].bodystruct 0 -datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -datatype[4].documenttype[0].importedfield[0].name "my_cool_field" -datatype[4].documenttype[0].importedfield[1].name "my_swag_field" -datatype[4].documenttype[0].importedfield[2].name "my_name" -datatype[5].id -2041471955 -datatype[5].structtype[0].name "campaign.header" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[5].structtype[0].field[0].name "cool_field" -datatype[5].structtype[0].field[0].datatype 2 -datatype[5].structtype[0].field[0].detailedtype "" -datatype[5].structtype[0].field[1].name "swag_field" -datatype[5].structtype[0].field[1].datatype 4 -datatype[5].structtype[0].field[1].detailedtype "" -datatype[6].id -1318255918 -datatype[6].documenttype[0].name "campaign" -datatype[6].documenttype[0].version 0 -datatype[6].documenttype[0].inherits[0].name "document" -datatype[6].documenttype[0].inherits[0].version 0 -datatype[6].documenttype[0].headerstruct -2041471955 -datatype[6].documenttype[0].bodystruct 0 -datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" -datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" -datatype[7].id 3129224 -datatype[7].structtype[0].name "person.header" -datatype[7].structtype[0].version 0 -datatype[7].structtype[0].compresstype NONE -datatype[7].structtype[0].compresslevel 0 -datatype[7].structtype[0].compressthreshold 95 -datatype[7].structtype[0].compressminsize 800 -datatype[7].structtype[0].field[0].name "name" -datatype[7].structtype[0].field[0].datatype 2 -datatype[7].structtype[0].field[0].detailedtype "" -datatype[8].id 443162583 -datatype[8].documenttype[0].name "person" -datatype[8].documenttype[0].version 0 -datatype[8].documenttype[0].inherits[0].name "document" -datatype[8].documenttype[0].inherits[0].version 0 -datatype[8].documenttype[0].headerstruct 3129224 -datatype[8].documenttype[0].bodystruct 0 -datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "name" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "ad" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[1].fieldsets{[document]}.fields[1] "person_ref" +doctype[1].importedfield[0].name "my_cool_field" +doctype[1].importedfield[1].name "my_swag_field" +doctype[1].importedfield[2].name "my_name" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[1].idx 10019 +doctype[1].documentref[1].targettype 10020 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "ad.header" +doctype[1].structtype[0].field[0].name "campaign_ref" +doctype[1].structtype[0].field[0].internalid 23963250 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "person_ref" +doctype[1].structtype[0].field[1].internalid 100779805 +doctype[1].structtype[0].field[1].type 10019 +doctype[2].name "campaign" +doctype[2].idx 10018 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10021 +doctype[2].fieldsets{[document]}.fields[0] "cool_field" +doctype[2].fieldsets{[document]}.fields[1] "swag_field" +doctype[2].structtype[0].idx 10021 +doctype[2].structtype[0].name "campaign.header" +doctype[2].structtype[0].field[0].name "cool_field" +doctype[2].structtype[0].field[0].internalid 1588702436 +doctype[2].structtype[0].field[0].type 10012 +doctype[2].structtype[0].field[1].name "swag_field" +doctype[2].structtype[0].field[1].internalid 1691224741 +doctype[2].structtype[0].field[1].type 10008 +doctype[3].name "person" +doctype[3].idx 10020 +doctype[3].inherits[0].idx 10000 +doctype[3].contentstruct 10022 +doctype[3].fieldsets{[document]}.fields[0] "name" +doctype[3].structtype[0].idx 10022 +doctype[3].structtype[0].name "person.header" +doctype[3].structtype[0].field[0].name "name" +doctype[3].structtype[0].field[0].internalid 1160796772 +doctype[3].structtype[0].field[0].type 10012 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg index d105b894b63..e69de29bb2d 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg @@ -1,35 +0,0 @@ -enablecompression false -usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1895788438 -datatype[].referencetype[].target_type_id 2987301 -datatype[].id 959075962 -datatype[].structtype[].name "ad.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "self_ref" -datatype[].structtype[].field[].datatype -1895788438 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 2987301 -datatype[].documenttype[].name "ad" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 959075962 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "self_ref" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index d7c9ddf8a70..48348348989 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -1,69 +1,71 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id 595216861 -datatype[1].referencetype[0].target_type_id -1318255918 -datatype[2].id 542332920 -datatype[2].referencetype[0].target_type_id 443162583 -datatype[3].id 959075962 -datatype[3].structtype[0].name "ad.header" -datatype[3].structtype[0].version 0 -datatype[3].structtype[0].compresstype NONE -datatype[3].structtype[0].compresslevel 0 -datatype[3].structtype[0].compressthreshold 95 -datatype[3].structtype[0].compressminsize 800 -datatype[3].structtype[0].field[0].name "campaign_ref" -datatype[3].structtype[0].field[0].datatype 595216861 -datatype[3].structtype[0].field[0].detailedtype "" -datatype[3].structtype[0].field[1].name "person_ref" -datatype[3].structtype[0].field[1].datatype 542332920 -datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id 2987301 -datatype[4].documenttype[0].name "ad" -datatype[4].documenttype[0].version 0 -datatype[4].documenttype[0].inherits[0].name "document" -datatype[4].documenttype[0].inherits[0].version 0 -datatype[4].documenttype[0].headerstruct 959075962 -datatype[4].documenttype[0].bodystruct 0 -datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -datatype[5].id -2041471955 -datatype[5].structtype[0].name "campaign.header" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id -1318255918 -datatype[6].documenttype[0].name "campaign" -datatype[6].documenttype[0].version 0 -datatype[6].documenttype[0].inherits[0].name "document" -datatype[6].documenttype[0].inherits[0].version 0 -datatype[6].documenttype[0].headerstruct -2041471955 -datatype[6].documenttype[0].bodystruct 0 -datatype[7].id 3129224 -datatype[7].structtype[0].name "person.header" -datatype[7].structtype[0].version 0 -datatype[7].structtype[0].compresstype NONE -datatype[7].structtype[0].compresslevel 0 -datatype[7].structtype[0].compressthreshold 95 -datatype[7].structtype[0].compressminsize 800 -datatype[8].id 443162583 -datatype[8].documenttype[0].name "person" -datatype[8].documenttype[0].version 0 -datatype[8].documenttype[0].inherits[0].name "document" -datatype[8].documenttype[0].inherits[0].version 0 -datatype[8].documenttype[0].headerstruct 3129224 -datatype[8].documenttype[0].bodystruct 0 +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "ad" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[1].fieldsets{[document]}.fields[1] "person_ref" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[1].idx 10019 +doctype[1].documentref[1].targettype 10020 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "ad.header" +doctype[1].structtype[0].field[0].name "campaign_ref" +doctype[1].structtype[0].field[0].internalid 23963250 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "person_ref" +doctype[1].structtype[0].field[1].internalid 100779805 +doctype[1].structtype[0].field[1].type 10019 +doctype[2].name "campaign" +doctype[2].idx 10018 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10021 +doctype[2].structtype[0].idx 10021 +doctype[2].structtype[0].name "campaign.header" +doctype[3].name "person" +doctype[3].idx 10020 +doctype[3].inherits[0].idx 10000 +doctype[3].contentstruct 10022 +doctype[3].structtype[0].idx 10022 +doctype[3].structtype[0].name "person.header" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index a99bac3a831..876ed00d0c4 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -1,53 +1,65 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id 595216861 -datatype[1].referencetype[0].target_type_id -1318255918 -datatype[2].id 959075962 -datatype[2].structtype[0].name "ad.header" -datatype[2].structtype[0].version 0 -datatype[2].structtype[0].compresstype NONE -datatype[2].structtype[0].compresslevel 0 -datatype[2].structtype[0].compressthreshold 95 -datatype[2].structtype[0].compressminsize 800 -datatype[2].structtype[0].field[0].name "campaign_ref" -datatype[2].structtype[0].field[0].datatype 595216861 -datatype[2].structtype[0].field[0].detailedtype "" -datatype[2].structtype[0].field[1].name "other_campaign_ref" -datatype[2].structtype[0].field[1].datatype 595216861 -datatype[2].structtype[0].field[1].detailedtype "" -datatype[3].id 2987301 -datatype[3].documenttype[0].name "ad" -datatype[3].documenttype[0].version 0 -datatype[3].documenttype[0].inherits[0].name "document" -datatype[3].documenttype[0].inherits[0].version 0 -datatype[3].documenttype[0].headerstruct 959075962 -datatype[3].documenttype[0].bodystruct 0 -datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[3].documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" -datatype[4].id -2041471955 -datatype[4].structtype[0].name "campaign.header" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[5].id -1318255918 -datatype[5].documenttype[0].name "campaign" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct -2041471955 -datatype[5].documenttype[0].bodystruct 0 +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "ad" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[1].idx 10019 +doctype[1].documentref[1].targettype 10018 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "ad.header" +doctype[1].structtype[0].field[0].name "campaign_ref" +doctype[1].structtype[0].field[0].internalid 23963250 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "other_campaign_ref" +doctype[1].structtype[0].field[1].internalid 874751172 +doctype[1].structtype[0].field[1].type 10019 +doctype[2].name "campaign" +doctype[2].idx 10018 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10020 +doctype[2].structtype[0].idx 10020 +doctype[2].structtype[0].name "campaign.header" diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index c317c19b09b..6eea5ae038b 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -1,86 +1,104 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1337915045 -datatype[].structtype[].name "advanced.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "debug_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "attributes_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "location_str" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "title_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "product_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "product2_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "product3_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "debug" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "attributes" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "title" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "product" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "product2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "product3" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "location_zcurve" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "title_s" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "mysummary" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 686681444 -datatype[].documenttype[].name "advanced" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -1337915045 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{titleabstract}.fields[] "title" -datatype[].documenttype[].fieldsets{default}.fields[] "title" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attributes_src" -datatype[].documenttype[].fieldsets{[document]}.fields[] "debug_src" -datatype[].documenttype[].fieldsets{[document]}.fields[] "location_str" -datatype[].documenttype[].fieldsets{[document]}.fields[] "product2_src" -datatype[].documenttype[].fieldsets{[document]}.fields[] "product3_src" -datatype[].documenttype[].fieldsets{[document]}.fields[] "product_src" -datatype[].documenttype[].fieldsets{[document]}.fields[] "title_src" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "advanced" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{titleabstract}.fields[] "title" +doctype[].fieldsets{default}.fields[] "title" +doctype[].fieldsets{[document]}.fields[] "attributes_src" +doctype[].fieldsets{[document]}.fields[] "debug_src" +doctype[].fieldsets{[document]}.fields[] "location_str" +doctype[].fieldsets{[document]}.fields[] "product2_src" +doctype[].fieldsets{[document]}.fields[] "product3_src" +doctype[].fieldsets{[document]}.fields[] "product_src" +doctype[].fieldsets{[document]}.fields[] "title_src" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "advanced.header" +doctype[].structtype[].field[].name "debug_src" +doctype[].structtype[].field[].internalid 2040219432 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "attributes_src" +doctype[].structtype[].field[].internalid 206068784 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "location_str" +doctype[].structtype[].field[].internalid 1878695266 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "title_src" +doctype[].structtype[].field[].internalid 1663435609 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "product_src" +doctype[].structtype[].field[].internalid 721375362 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "product2_src" +doctype[].structtype[].field[].internalid 1432937317 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "product3_src" +doctype[].structtype[].field[].internalid 1601223618 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "debug" +doctype[].structtype[].field[].internalid 1852175867 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "attributes" +doctype[].structtype[].field[].internalid 1807427837 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "title" +doctype[].structtype[].field[].internalid 567626448 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "product" +doctype[].structtype[].field[].internalid 516982522 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "product2" +doctype[].structtype[].field[].internalid 1822067147 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "product3" +doctype[].structtype[].field[].internalid 1855194930 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "location_zcurve" +doctype[].structtype[].field[].internalid 357924406 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "title_s" +doctype[].structtype[].field[].internalid 1424913045 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "mysummary" +doctype[].structtype[].field[].internalid 118911010 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index cdf554cb747..f15b49c6b76 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -1,42 +1,56 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 517946310 -datatype[].structtype[].name "annotation.banana" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "brand" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -364910881 -datatype[].structtype[].name "annotationsimplicitstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -2099544992 -datatype[].documenttype[].name "annotationsimplicitstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -364910881 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id -269517759 -annotationtype[].name "banana" -annotationtype[].datatype 517946310 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsimplicitstruct" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10018 +doctype[].annotationtype[].name "banana" +doctype[].annotationtype[].internalid -269517759 +doctype[].annotationtype[].datatype 10017 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsimplicitstruct.header" +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "annotation.banana" +doctype[].structtype[].field[].name "brand" +doctype[].structtype[].field[].internalid 2085058127 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index 9633eaa532b..4098c0fe53c 100644 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -1,133 +1,120 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 517946310 -datatype[].structtype[].name "annotation.banana" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "brand" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1339036621 -datatype[].structtype[].name "annotation.intern" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "enddate" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].inherits[].name "annotation.employee" -datatype[].structtype[].inherits[].version 0 -datatype[].id 249059607 -datatype[].structtype[].name "annotation.car" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "color" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].inherits[].name "annotation.vehicle" -datatype[].structtype[].inherits[].version 0 -datatype[].id -1466283082 -datatype[].structtype[].name "annotation.person" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "name" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -858216177 -datatype[].structtype[].name "annotation.employee" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "employeeid" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].inherits[].name "annotation.worker" -datatype[].structtype[].inherits[].version 0 -datatype[].id -1874092641 -datatype[].structtype[].name "annotation.worker" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].inherits[].name "annotation.person" -datatype[].structtype[].inherits[].version 0 -datatype[].id -1047410193 -datatype[].structtype[].name "annotation.vehicle" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "numwheels" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1406250281 -datatype[].structtype[].name "annotationsinheritance.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -748546200 -datatype[].documenttype[].name "annotationsinheritance" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -1406250281 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id -269517759 -annotationtype[].name "banana" -annotationtype[].datatype 517946310 -annotationtype[].inherits[].id 877283632 -annotationtype[].id 855102455 -annotationtype[].name "intern" -annotationtype[].datatype -1339036621 -annotationtype[].inherits[].id 804106508 -annotationtype[].id -973728295 -annotationtype[].name "car" -annotationtype[].datatype 249059607 -annotationtype[].inherits[].id 290814930 -annotationtype[].id 877283632 -annotationtype[].name "fruit" -annotationtype[].datatype -1 -annotationtype[].id 609952424 -annotationtype[].name "person" -annotationtype[].datatype -1466283082 -annotationtype[].id 804106508 -annotationtype[].name "employee" -annotationtype[].datatype -858216177 -annotationtype[].inherits[].id 881692980 -annotationtype[].id 881692980 -annotationtype[].name "worker" -annotationtype[].datatype -1874092641 -annotationtype[].inherits[].id 609952424 -annotationtype[].id 290814930 -annotationtype[].name "vehicle" -annotationtype[].datatype -1047410193 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsinheritance" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10024 +doctype[].annotationtype[].name "banana" +doctype[].annotationtype[].internalid -269517759 +doctype[].annotationtype[].datatype 10017 +doctype[].annotationtype[].inherits[].idx 10025 +doctype[].annotationtype[].idx 10026 +doctype[].annotationtype[].name "car" +doctype[].annotationtype[].internalid -973728295 +doctype[].annotationtype[].datatype 10018 +doctype[].annotationtype[].inherits[].idx 10027 +doctype[].annotationtype[].idx 10028 +doctype[].annotationtype[].name "employee" +doctype[].annotationtype[].internalid 804106508 +doctype[].annotationtype[].datatype 10020 +doctype[].annotationtype[].inherits[].idx 10029 +doctype[].annotationtype[].idx 10025 +doctype[].annotationtype[].name "fruit" +doctype[].annotationtype[].internalid 877283632 +doctype[].annotationtype[].datatype -1 +doctype[].annotationtype[].idx 10030 +doctype[].annotationtype[].name "intern" +doctype[].annotationtype[].internalid 855102455 +doctype[].annotationtype[].datatype 10023 +doctype[].annotationtype[].inherits[].idx 10028 +doctype[].annotationtype[].idx 10031 +doctype[].annotationtype[].name "person" +doctype[].annotationtype[].internalid 609952424 +doctype[].annotationtype[].datatype 10022 +doctype[].annotationtype[].idx 10027 +doctype[].annotationtype[].name "vehicle" +doctype[].annotationtype[].internalid 290814930 +doctype[].annotationtype[].datatype 10019 +doctype[].annotationtype[].idx 10029 +doctype[].annotationtype[].name "worker" +doctype[].annotationtype[].internalid 881692980 +doctype[].annotationtype[].datatype 10021 +doctype[].annotationtype[].inherits[].idx 10031 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsinheritance.header" +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "annotation.banana" +doctype[].structtype[].field[].name "brand" +doctype[].structtype[].field[].internalid 2085058127 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "annotation.vehicle" +doctype[].structtype[].field[].name "numwheels" +doctype[].structtype[].field[].internalid 50890304 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "annotation.car" +doctype[].structtype[].inherits[].type 10019 +doctype[].structtype[].field[].name "color" +doctype[].structtype[].field[].internalid 344415742 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10022 +doctype[].structtype[].name "annotation.person" +doctype[].structtype[].field[].name "name" +doctype[].structtype[].field[].internalid 1160796772 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10021 +doctype[].structtype[].name "annotation.worker" +doctype[].structtype[].inherits[].type 10022 +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "annotation.employee" +doctype[].structtype[].inherits[].type 10021 +doctype[].structtype[].field[].name "employeeid" +doctype[].structtype[].field[].internalid 109528570 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].idx 10023 +doctype[].structtype[].name "annotation.intern" +doctype[].structtype[].inherits[].type 10020 +doctype[].structtype[].field[].name "enddate" +doctype[].structtype[].field[].internalid 206146600 +doctype[].structtype[].field[].type 10008 diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index 85aef02bb3c..b67f1df3b4b 100644 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -1,91 +1,94 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1443831334 -datatype[].structtype[].name "annotation.c" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "cfoo" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1443832295 -datatype[].structtype[].name "annotation.d" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].inherits[].name "annotation.c" -datatype[].structtype[].inherits[].version 0 -datatype[].id 1443833256 -datatype[].structtype[].name "annotation.e" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].inherits[].name "annotation.d" -datatype[].structtype[].inherits[].version 0 -datatype[].id 1443835178 -datatype[].structtype[].name "annotation.g" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "gfoo" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 424382193 -datatype[].structtype[].name "annotationsinheritance2.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -1730091890 -datatype[].documenttype[].name "annotationsinheritance2" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 424382193 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id 1769416289 -annotationtype[].name "a" -annotationtype[].datatype -1 -annotationtype[].id 1966167951 -annotationtype[].name "b" -annotationtype[].datatype -1 -annotationtype[].inherits[].id 1769416289 -annotationtype[].id 1082875699 -annotationtype[].name "c" -annotationtype[].datatype 1443831334 -annotationtype[].id 383816109 -annotationtype[].name "d" -annotationtype[].datatype 1443832295 -annotationtype[].inherits[].id 1082875699 -annotationtype[].id -398332878 -annotationtype[].name "e" -annotationtype[].datatype 1443833256 -annotationtype[].inherits[].id 383816109 -annotationtype[].id 422169831 -annotationtype[].name "f" -annotationtype[].datatype -1 -annotationtype[].id 907314269 -annotationtype[].name "g" -annotationtype[].datatype 1443835178 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsinheritance2" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10021 +doctype[].annotationtype[].name "a" +doctype[].annotationtype[].internalid 1769416289 +doctype[].annotationtype[].datatype -1 +doctype[].annotationtype[].idx 10022 +doctype[].annotationtype[].name "b" +doctype[].annotationtype[].internalid 1966167951 +doctype[].annotationtype[].datatype -1 +doctype[].annotationtype[].inherits[].idx 10021 +doctype[].annotationtype[].idx 10023 +doctype[].annotationtype[].name "c" +doctype[].annotationtype[].internalid 1082875699 +doctype[].annotationtype[].datatype 10017 +doctype[].annotationtype[].idx 10024 +doctype[].annotationtype[].name "d" +doctype[].annotationtype[].internalid 383816109 +doctype[].annotationtype[].datatype 10018 +doctype[].annotationtype[].inherits[].idx 10023 +doctype[].annotationtype[].idx 10025 +doctype[].annotationtype[].name "e" +doctype[].annotationtype[].internalid -398332878 +doctype[].annotationtype[].datatype 10019 +doctype[].annotationtype[].inherits[].idx 10024 +doctype[].annotationtype[].idx 10026 +doctype[].annotationtype[].name "f" +doctype[].annotationtype[].internalid 422169831 +doctype[].annotationtype[].datatype -1 +doctype[].annotationtype[].idx 10027 +doctype[].annotationtype[].name "g" +doctype[].annotationtype[].internalid 907314269 +doctype[].annotationtype[].datatype 10020 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsinheritance2.header" +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "annotation.c" +doctype[].structtype[].field[].name "cfoo" +doctype[].structtype[].field[].internalid 648705706 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "annotation.d" +doctype[].structtype[].inherits[].type 10017 +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "annotation.e" +doctype[].structtype[].inherits[].type 10018 +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "annotation.g" +doctype[].structtype[].field[].name "gfoo" +doctype[].structtype[].field[].internalid 1703390249 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index a8d46f2a940..df8d038fe79 100644 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -1,51 +1,67 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -2014701853 -datatype[].annotationreftype[].annotation "super" -datatype[].id -888007918 -datatype[].structtype[].name "annotation.blah" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "a" -datatype[].structtype[].field[].datatype -2014701853 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1552577796 -datatype[].structtype[].name "annotationspolymorphy.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -1383624989 -datatype[].documenttype[].name "annotationspolymorphy" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -1552577796 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id 668095690 -annotationtype[].name "super" -annotationtype[].datatype -1 -annotationtype[].id 119710016 -annotationtype[].name "sub" -annotationtype[].datatype -1 -annotationtype[].inherits[].id 668095690 -annotationtype[].id -1793776935 -annotationtype[].name "blah" -annotationtype[].datatype -888007918 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationspolymorphy" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10020 +doctype[].annotationtype[].name "blah" +doctype[].annotationtype[].internalid -1793776935 +doctype[].annotationtype[].datatype 10017 +doctype[].annotationtype[].idx 10021 +doctype[].annotationtype[].name "sub" +doctype[].annotationtype[].internalid 119710016 +doctype[].annotationtype[].datatype -1 +doctype[].annotationtype[].inherits[].idx 10019 +doctype[].annotationtype[].idx 10019 +doctype[].annotationtype[].name "super" +doctype[].annotationtype[].internalid 668095690 +doctype[].annotationtype[].datatype -1 +doctype[].annotationref[].idx 10018 +doctype[].annotationref[].annotationtype 10019 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationspolymorphy.header" +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "annotation.blah" +doctype[].structtype[].field[].name "a" +doctype[].structtype[].field[].internalid 1756498266 +doctype[].structtype[].field[].type 10018 diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index 7ce09a97be1..94258fdb798 100644 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -1,90 +1,93 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 517946310 -datatype[].structtype[].name "annotation.banana" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "brand" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 400622300 -datatype[].annotationreftype[].annotation "b" -datatype[].id 1443829412 -datatype[].structtype[].name "annotation.a" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "foo" -datatype[].structtype[].field[].datatype 400622300 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -808460615 -datatype[].annotationreftype[].annotation "banana" -datatype[].id -770307521 -datatype[].structtype[].name "annotation.food" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "what" -datatype[].structtype[].field[].datatype -808460615 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 756306917 -datatype[].annotationreftype[].annotation "cyclic" -datatype[].id 1781099546 -datatype[].structtype[].name "annotation.cyclic" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "blah" -datatype[].structtype[].field[].datatype 756306917 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 571255414 -datatype[].structtype[].name "annotationsreference.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -1448377175 -datatype[].documenttype[].name "annotationsreference" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 571255414 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id -269517759 -annotationtype[].name "banana" -annotationtype[].datatype 517946310 -annotationtype[].id 1769416289 -annotationtype[].name "a" -annotationtype[].datatype 1443829412 -annotationtype[].id 1966167951 -annotationtype[].name "b" -annotationtype[].datatype -1 -annotationtype[].id 1918102169 -annotationtype[].name "food" -annotationtype[].datatype -770307521 -annotationtype[].id -1569831750 -annotationtype[].name "cyclic" -annotationtype[].datatype 1781099546 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsreference" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10027 +doctype[].annotationtype[].name "a" +doctype[].annotationtype[].internalid 1769416289 +doctype[].annotationtype[].datatype 10017 +doctype[].annotationtype[].idx 10019 +doctype[].annotationtype[].name "b" +doctype[].annotationtype[].internalid 1966167951 +doctype[].annotationtype[].datatype -1 +doctype[].annotationtype[].idx 10026 +doctype[].annotationtype[].name "banana" +doctype[].annotationtype[].internalid -269517759 +doctype[].annotationtype[].datatype 10020 +doctype[].annotationtype[].idx 10023 +doctype[].annotationtype[].name "cyclic" +doctype[].annotationtype[].internalid -1569831750 +doctype[].annotationtype[].datatype 10021 +doctype[].annotationtype[].idx 10028 +doctype[].annotationtype[].name "food" +doctype[].annotationtype[].internalid 1918102169 +doctype[].annotationtype[].datatype 10024 +doctype[].annotationref[].idx 10018 +doctype[].annotationref[].annotationtype 10019 +doctype[].annotationref[].idx 10022 +doctype[].annotationref[].annotationtype 10023 +doctype[].annotationref[].idx 10025 +doctype[].annotationref[].annotationtype 10026 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsreference.header" +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "annotation.a" +doctype[].structtype[].field[].name "foo" +doctype[].structtype[].field[].internalid 1979988295 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "annotation.banana" +doctype[].structtype[].field[].name "brand" +doctype[].structtype[].field[].internalid 2085058127 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10021 +doctype[].structtype[].name "annotation.cyclic" +doctype[].structtype[].field[].name "blah" +doctype[].structtype[].field[].internalid 874732190 +doctype[].structtype[].field[].type 10022 +doctype[].structtype[].idx 10024 +doctype[].structtype[].name "annotation.food" +doctype[].structtype[].field[].name "what" +doctype[].structtype[].field[].internalid 445721780 +doctype[].structtype[].field[].type 10025 diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index 1342a179239..e863e305643 100644 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -1,32 +1,51 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1205708249 -datatype[].structtype[].name "annotationssimple.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -1584092648 -datatype[].documenttype[].name "annotationssimple" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -1205708249 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id -269517759 -annotationtype[].name "banana" -annotationtype[].datatype -1 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationssimple" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10017 +doctype[].annotationtype[].name "banana" +doctype[].annotationtype[].internalid -269517759 +doctype[].annotationtype[].datatype -1 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationssimple.header" diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index 233c2f729fe..62084994721 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -1,52 +1,61 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1293792650 -datatype[].structtype[].name "my_struct" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "my_structfield" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1080124700 -datatype[].structtype[].name "annotation.my_anno" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "my_annofield" -datatype[].structtype[].field[].datatype 1293792650 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1341437796 -datatype[].structtype[].name "annotationsstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -263977093 -datatype[].documenttype[].name "annotationsstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 1341437796 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id -160036815 -annotationtype[].name "my_anno" -annotationtype[].datatype -1080124700 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsstruct" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].annotationtype[].idx 10019 +doctype[].annotationtype[].name "my_anno" +doctype[].annotationtype[].internalid -160036815 +doctype[].annotationtype[].datatype 10017 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsstruct.header" +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "my_struct" +doctype[].structtype[].field[].name "my_structfield" +doctype[].structtype[].field[].internalid 327607926 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "annotation.my_anno" +doctype[].structtype[].field[].name "my_annofield" +doctype[].structtype[].field[].internalid 594209253 +doctype[].structtype[].field[].type 10018 diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index 19c1c5eda2f..640655c53b6 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -1,54 +1,63 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1293792650 -datatype[].structtype[].name "my_struct" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "my_structfield" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 754837689 -datatype[].arraytype[].datatype 1293792650 -datatype[].id -1080124700 -datatype[].structtype[].name "annotation.my_anno" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "my_annofield" -datatype[].structtype[].field[].datatype 754837689 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 94945597 -datatype[].structtype[].name "annotationsstructarray.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id 1593733058 -datatype[].documenttype[].name "annotationsstructarray" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 94945597 -datatype[].documenttype[].bodystruct 0 -annotationtype[].id -160036815 -annotationtype[].name "my_anno" -annotationtype[].datatype -1080124700 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsstructarray" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10018 +doctype[].annotationtype[].idx 10020 +doctype[].annotationtype[].name "my_anno" +doctype[].annotationtype[].internalid -160036815 +doctype[].annotationtype[].datatype 10019 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsstructarray.header" +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "my_struct" +doctype[].structtype[].field[].name "my_structfield" +doctype[].structtype[].field[].internalid 327607926 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "annotation.my_anno" +doctype[].structtype[].field[].name "my_annofield" +doctype[].structtype[].field[].internalid 718266337 +doctype[].structtype[].field[].type 10017 diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index 554cf017b54..7da3b44f43c 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -1,60 +1,80 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1486737430 -datatype[].arraytype[].datatype 2 -datatype[].id -1245117006 -datatype[].arraytype[].datatype 0 -datatype[].id 1328286588 -datatype[].weightedsettype[].datatype 2 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id 1081627459 -datatype[].structtype[].name "arrays.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "tags" -datatype[].structtype[].field[].datatype -1486737430 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "ratings" -datatype[].structtype[].field[].datatype -1245117006 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "a" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "b" -datatype[].structtype[].field[].datatype -1486737430 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c" -datatype[].structtype[].field[].datatype 1328286588 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1292863364 -datatype[].documenttype[].name "arrays" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 1081627459 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{default}.fields[] "a" -datatype[].documenttype[].fieldsets{default}.fields[] "b" -datatype[].documenttype[].fieldsets{default}.fields[] "c" -datatype[].documenttype[].fieldsets{[document]}.fields[] "a" -datatype[].documenttype[].fieldsets{[document]}.fields[] "b" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c" -datatype[].documenttype[].fieldsets{[document]}.fields[] "ratings" -datatype[].documenttype[].fieldsets{[document]}.fields[] "tags" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "arrays" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{default}.fields[] "a" +doctype[].fieldsets{default}.fields[] "b" +doctype[].fieldsets{default}.fields[] "c" +doctype[].fieldsets{[document]}.fields[] "a" +doctype[].fieldsets{[document]}.fields[] "b" +doctype[].fieldsets{[document]}.fields[] "c" +doctype[].fieldsets{[document]}.fields[] "ratings" +doctype[].fieldsets{[document]}.fields[] "tags" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].idx 10018 +doctype[].arraytype[].elementtype 10007 +doctype[].arraytype[].idx 10019 +doctype[].arraytype[].elementtype 10012 +doctype[].wsettype[].idx 10020 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "arrays.header" +doctype[].structtype[].field[].name "tags" +doctype[].structtype[].field[].internalid 477541661 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "ratings" +doctype[].structtype[].field[].internalid 80274180 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "a" +doctype[].structtype[].field[].internalid 493339625 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "b" +doctype[].structtype[].field[].internalid 1913648692 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "c" +doctype[].structtype[].field[].internalid 2069269023 +doctype[].structtype[].field[].type 10020 diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index b26698d83a6..373e0211253 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -1,137 +1,155 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 49942803 -datatype[].arraytype[].datatype 16 -datatype[].id -1068914395 -datatype[].weightedsettype[].datatype 16 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id -1245117006 -datatype[].arraytype[].datatype 0 -datatype[].id 519906144 -datatype[].weightedsettype[].datatype 0 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id 58874399 -datatype[].arraytype[].datatype 4 -datatype[].id -1059982799 -datatype[].weightedsettype[].datatype 4 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id 1650586661 -datatype[].arraytype[].datatype 1 -datatype[].id 1325751891 -datatype[].weightedsettype[].datatype 1 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id -2054976470 -datatype[].arraytype[].datatype 5 -datatype[].id 760047548 -datatype[].weightedsettype[].datatype 5 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id -1486737430 -datatype[].arraytype[].datatype 2 -datatype[].id 1328286588 -datatype[].weightedsettype[].datatype 2 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id -109105370 -datatype[].structtype[].name "prefetch.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "singlebyte" -datatype[].structtype[].field[].datatype 16 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "multibyte" -datatype[].structtype[].field[].datatype 49942803 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "wsbyte" -datatype[].structtype[].field[].datatype -1068914395 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "singleint" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "multiint" -datatype[].structtype[].field[].datatype -1245117006 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "wsint" -datatype[].structtype[].field[].datatype 519906144 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "singlelong" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "multilong" -datatype[].structtype[].field[].datatype 58874399 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "wslong" -datatype[].structtype[].field[].datatype -1059982799 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "singlefloat" -datatype[].structtype[].field[].datatype 1 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "multifloat" -datatype[].structtype[].field[].datatype 1650586661 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "wsfloat" -datatype[].structtype[].field[].datatype 1325751891 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "singledouble" -datatype[].structtype[].field[].datatype 5 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "multidouble" -datatype[].structtype[].field[].datatype -2054976470 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "wsdouble" -datatype[].structtype[].field[].datatype 760047548 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "singlestring" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "multistring" -datatype[].structtype[].field[].datatype -1486737430 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "wsstring" -datatype[].structtype[].field[].datatype 1328286588 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1458051591 -datatype[].documenttype[].name "prefetch" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -109105370 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "multibyte" -datatype[].documenttype[].fieldsets{[document]}.fields[] "multidouble" -datatype[].documenttype[].fieldsets{[document]}.fields[] "multifloat" -datatype[].documenttype[].fieldsets{[document]}.fields[] "multiint" -datatype[].documenttype[].fieldsets{[document]}.fields[] "multilong" -datatype[].documenttype[].fieldsets{[document]}.fields[] "multistring" -datatype[].documenttype[].fieldsets{[document]}.fields[] "singlebyte" -datatype[].documenttype[].fieldsets{[document]}.fields[] "singledouble" -datatype[].documenttype[].fieldsets{[document]}.fields[] "singlefloat" -datatype[].documenttype[].fieldsets{[document]}.fields[] "singleint" -datatype[].documenttype[].fieldsets{[document]}.fields[] "singlelong" -datatype[].documenttype[].fieldsets{[document]}.fields[] "singlestring" -datatype[].documenttype[].fieldsets{[document]}.fields[] "wsbyte" -datatype[].documenttype[].fieldsets{[document]}.fields[] "wsdouble" -datatype[].documenttype[].fieldsets{[document]}.fields[] "wsfloat" -datatype[].documenttype[].fieldsets{[document]}.fields[] "wsint" -datatype[].documenttype[].fieldsets{[document]}.fields[] "wslong" -datatype[].documenttype[].fieldsets{[document]}.fields[] "wsstring" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "prefetch" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "multibyte" +doctype[].fieldsets{[document]}.fields[] "multidouble" +doctype[].fieldsets{[document]}.fields[] "multifloat" +doctype[].fieldsets{[document]}.fields[] "multiint" +doctype[].fieldsets{[document]}.fields[] "multilong" +doctype[].fieldsets{[document]}.fields[] "multistring" +doctype[].fieldsets{[document]}.fields[] "singlebyte" +doctype[].fieldsets{[document]}.fields[] "singledouble" +doctype[].fieldsets{[document]}.fields[] "singlefloat" +doctype[].fieldsets{[document]}.fields[] "singleint" +doctype[].fieldsets{[document]}.fields[] "singlelong" +doctype[].fieldsets{[document]}.fields[] "singlestring" +doctype[].fieldsets{[document]}.fields[] "wsbyte" +doctype[].fieldsets{[document]}.fields[] "wsdouble" +doctype[].fieldsets{[document]}.fields[] "wsfloat" +doctype[].fieldsets{[document]}.fields[] "wsint" +doctype[].fieldsets{[document]}.fields[] "wslong" +doctype[].fieldsets{[document]}.fields[] "wsstring" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10003 +doctype[].arraytype[].idx 10019 +doctype[].arraytype[].elementtype 10007 +doctype[].arraytype[].idx 10021 +doctype[].arraytype[].elementtype 10008 +doctype[].arraytype[].idx 10023 +doctype[].arraytype[].elementtype 10005 +doctype[].arraytype[].idx 10025 +doctype[].arraytype[].elementtype 10004 +doctype[].arraytype[].idx 10027 +doctype[].arraytype[].elementtype 10012 +doctype[].wsettype[].idx 10018 +doctype[].wsettype[].elementtype 10003 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10020 +doctype[].wsettype[].elementtype 10007 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10022 +doctype[].wsettype[].elementtype 10008 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10024 +doctype[].wsettype[].elementtype 10005 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10026 +doctype[].wsettype[].elementtype 10004 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10028 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "prefetch.header" +doctype[].structtype[].field[].name "singlebyte" +doctype[].structtype[].field[].internalid 283133197 +doctype[].structtype[].field[].type 10003 +doctype[].structtype[].field[].name "multibyte" +doctype[].structtype[].field[].internalid 97388815 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "wsbyte" +doctype[].structtype[].field[].internalid 208948697 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "singleint" +doctype[].structtype[].field[].internalid 939533053 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "multiint" +doctype[].structtype[].field[].internalid 1746431344 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "wsint" +doctype[].structtype[].field[].internalid 1936248443 +doctype[].structtype[].field[].type 10020 +doctype[].structtype[].field[].name "singlelong" +doctype[].structtype[].field[].internalid 926038302 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "multilong" +doctype[].structtype[].field[].internalid 1450042501 +doctype[].structtype[].field[].type 10021 +doctype[].structtype[].field[].name "wslong" +doctype[].structtype[].field[].internalid 838342430 +doctype[].structtype[].field[].type 10022 +doctype[].structtype[].field[].name "singlefloat" +doctype[].structtype[].field[].internalid 454654240 +doctype[].structtype[].field[].type 10005 +doctype[].structtype[].field[].name "multifloat" +doctype[].structtype[].field[].internalid 1028626753 +doctype[].structtype[].field[].type 10023 +doctype[].structtype[].field[].name "wsfloat" +doctype[].structtype[].field[].internalid 2087992058 +doctype[].structtype[].field[].type 10024 +doctype[].structtype[].field[].name "singledouble" +doctype[].structtype[].field[].internalid 1982688634 +doctype[].structtype[].field[].type 10004 +doctype[].structtype[].field[].name "multidouble" +doctype[].structtype[].field[].internalid 1316159002 +doctype[].structtype[].field[].type 10025 +doctype[].structtype[].field[].name "wsdouble" +doctype[].structtype[].field[].internalid 995331392 +doctype[].structtype[].field[].type 10026 +doctype[].structtype[].field[].name "singlestring" +doctype[].structtype[].field[].internalid 289408547 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "multistring" +doctype[].structtype[].field[].internalid 862051908 +doctype[].structtype[].field[].type 10027 +doctype[].structtype[].field[].name "wsstring" +doctype[].structtype[].field[].internalid 447961272 +doctype[].structtype[].field[].type 10028 diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index bc5947ad2b5..da4f2fc0942 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -1,136 +1,154 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1650586661 -datatype[].arraytype[].datatype 1 -datatype[].id -1245117006 -datatype[].arraytype[].datatype 0 -datatype[].id -1749463923 -datatype[].structtype[].name "complex.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "title" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "location" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "dyntitle" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "special1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "special2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "special3" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "prefixenabled" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "source" -datatype[].structtype[].field[].datatype 10 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "docurl" -datatype[].structtype[].field[].datatype 10 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "fleeting" -datatype[].structtype[].field[].datatype 1650586661 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "fleeting2" -datatype[].structtype[].field[].datatype 1 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "foundat" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "collapseby" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "yEaR" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "stringfield" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "exactemento" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "exactagain" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "ts" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "combineda" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "combinedb" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "category" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "woe" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "year_sub" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "year_arr" -datatype[].structtype[].field[].datatype -1245117006 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "exact" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1402929550 -datatype[].documenttype[].name "complex" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -1749463923 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{default}.fields[] "stringfield" -datatype[].documenttype[].fieldsets{default}.fields[] "title" -datatype[].documenttype[].fieldsets{special}.fields[] "special1" -datatype[].documenttype[].fieldsets{special}.fields[] "special2" -datatype[].documenttype[].fieldsets{special}.fields[] "special3" -datatype[].documenttype[].fieldsets{all}.fields[] "combineda" -datatype[].documenttype[].fieldsets{all}.fields[] "combinedb" -datatype[].documenttype[].fieldsets{[document]}.fields[] "category" -datatype[].documenttype[].fieldsets{[document]}.fields[] "collapseby" -datatype[].documenttype[].fieldsets{[document]}.fields[] "combineda" -datatype[].documenttype[].fieldsets{[document]}.fields[] "combinedb" -datatype[].documenttype[].fieldsets{[document]}.fields[] "docurl" -datatype[].documenttype[].fieldsets{[document]}.fields[] "dyntitle" -datatype[].documenttype[].fieldsets{[document]}.fields[] "exactagain" -datatype[].documenttype[].fieldsets{[document]}.fields[] "exactemento" -datatype[].documenttype[].fieldsets{[document]}.fields[] "fleeting" -datatype[].documenttype[].fieldsets{[document]}.fields[] "fleeting2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "foundat" -datatype[].documenttype[].fieldsets{[document]}.fields[] "location" -datatype[].documenttype[].fieldsets{[document]}.fields[] "prefixenabled" -datatype[].documenttype[].fieldsets{[document]}.fields[] "source" -datatype[].documenttype[].fieldsets{[document]}.fields[] "special1" -datatype[].documenttype[].fieldsets{[document]}.fields[] "special2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "special3" -datatype[].documenttype[].fieldsets{[document]}.fields[] "stringfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "title" -datatype[].documenttype[].fieldsets{[document]}.fields[] "ts" -datatype[].documenttype[].fieldsets{[document]}.fields[] "yEaR" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "complex" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{default}.fields[] "stringfield" +doctype[].fieldsets{default}.fields[] "title" +doctype[].fieldsets{special}.fields[] "special1" +doctype[].fieldsets{special}.fields[] "special2" +doctype[].fieldsets{special}.fields[] "special3" +doctype[].fieldsets{all}.fields[] "combineda" +doctype[].fieldsets{all}.fields[] "combinedb" +doctype[].fieldsets{[document]}.fields[] "category" +doctype[].fieldsets{[document]}.fields[] "collapseby" +doctype[].fieldsets{[document]}.fields[] "combineda" +doctype[].fieldsets{[document]}.fields[] "combinedb" +doctype[].fieldsets{[document]}.fields[] "docurl" +doctype[].fieldsets{[document]}.fields[] "dyntitle" +doctype[].fieldsets{[document]}.fields[] "exactagain" +doctype[].fieldsets{[document]}.fields[] "exactemento" +doctype[].fieldsets{[document]}.fields[] "fleeting" +doctype[].fieldsets{[document]}.fields[] "fleeting2" +doctype[].fieldsets{[document]}.fields[] "foundat" +doctype[].fieldsets{[document]}.fields[] "location" +doctype[].fieldsets{[document]}.fields[] "prefixenabled" +doctype[].fieldsets{[document]}.fields[] "source" +doctype[].fieldsets{[document]}.fields[] "special1" +doctype[].fieldsets{[document]}.fields[] "special2" +doctype[].fieldsets{[document]}.fields[] "special3" +doctype[].fieldsets{[document]}.fields[] "stringfield" +doctype[].fieldsets{[document]}.fields[] "title" +doctype[].fieldsets{[document]}.fields[] "ts" +doctype[].fieldsets{[document]}.fields[] "yEaR" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10005 +doctype[].arraytype[].idx 10018 +doctype[].arraytype[].elementtype 10007 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "complex.header" +doctype[].structtype[].field[].name "title" +doctype[].structtype[].field[].internalid 567626448 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "location" +doctype[].structtype[].field[].internalid 2099439365 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "dyntitle" +doctype[].structtype[].field[].internalid 182615872 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "special1" +doctype[].structtype[].field[].internalid 37251308 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "special2" +doctype[].structtype[].field[].internalid 46142511 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "special3" +doctype[].structtype[].field[].internalid 605997964 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "prefixenabled" +doctype[].structtype[].field[].internalid 1688590618 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "source" +doctype[].structtype[].field[].internalid 563409865 +doctype[].structtype[].field[].type 10014 +doctype[].structtype[].field[].name "docurl" +doctype[].structtype[].field[].internalid 689738609 +doctype[].structtype[].field[].type 10014 +doctype[].structtype[].field[].name "fleeting" +doctype[].structtype[].field[].internalid 1278868010 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "fleeting2" +doctype[].structtype[].field[].internalid 298469070 +doctype[].structtype[].field[].type 10005 +doctype[].structtype[].field[].name "foundat" +doctype[].structtype[].field[].internalid 827329375 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "collapseby" +doctype[].structtype[].field[].internalid 2107367198 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "yEaR" +doctype[].structtype[].field[].internalid 1488364441 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "stringfield" +doctype[].structtype[].field[].internalid 1182460484 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "exactemento" +doctype[].structtype[].field[].internalid 1641665351 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "exactagain" +doctype[].structtype[].field[].internalid 2115261858 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "ts" +doctype[].structtype[].field[].internalid 227578142 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "combineda" +doctype[].structtype[].field[].internalid 1941849266 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "combinedb" +doctype[].structtype[].field[].internalid 215566953 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "category" +doctype[].structtype[].field[].internalid 1356821726 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "woe" +doctype[].structtype[].field[].internalid 730907325 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "year_sub" +doctype[].structtype[].field[].internalid 920405828 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "year_arr" +doctype[].structtype[].field[].internalid 2008361478 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "exact" +doctype[].structtype[].field[].internalid 1658433671 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/declstruct/bar.sd b/config-model/src/test/derived/declstruct/bar.sd new file mode 100644 index 00000000000..7f7d06d5af7 --- /dev/null +++ b/config-model/src/test/derived/declstruct/bar.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search bar { + document bar inherits common { + struct mystructinbar { + field f2 type string { } + } + } +} diff --git a/config-model/src/test/derived/declstruct/common.sd b/config-model/src/test/derived/declstruct/common.sd new file mode 100644 index 00000000000..dc691fb4ac7 --- /dev/null +++ b/config-model/src/test/derived/declstruct/common.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search common { + document common { + struct mystruct { + field f0 type string { } + } + } +} diff --git a/config-model/src/test/derived/declstruct/documentmanager.cfg b/config-model/src/test/derived/declstruct/documentmanager.cfg new file mode 100644 index 00000000000..d9c48ddc42f --- /dev/null +++ b/config-model/src/test/derived/declstruct/documentmanager.cfg @@ -0,0 +1,89 @@ +enablecompression false +usev8geopositions false +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "common" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "common.header" +doctype[1].structtype[1].idx 10017 +doctype[1].structtype[1].name "mystruct" +doctype[1].structtype[1].field[0].name "f0" +doctype[1].structtype[1].field[0].internalid 111558427 +doctype[1].structtype[1].field[0].type 10012 +doctype[2].name "foo" +doctype[2].idx 10018 +doctype[2].inherits[0].idx 10000 +doctype[2].inherits[1].idx 10015 +doctype[2].contentstruct 10019 +doctype[2].structtype[0].idx 10019 +doctype[2].structtype[0].name "foo.header" +doctype[2].structtype[1].idx 10020 +doctype[2].structtype[1].name "mystructinfoo" +doctype[2].structtype[1].field[0].name "f1" +doctype[2].structtype[1].field[0].internalid 1911889118 +doctype[2].structtype[1].field[0].type 10012 +doctype[3].name "bar" +doctype[3].idx 10021 +doctype[3].inherits[0].idx 10000 +doctype[3].inherits[1].idx 10015 +doctype[3].contentstruct 10022 +doctype[3].structtype[0].idx 10022 +doctype[3].structtype[0].name "bar.header" +doctype[3].structtype[1].idx 10023 +doctype[3].structtype[1].name "mystructinbar" +doctype[3].structtype[1].field[0].name "f2" +doctype[3].structtype[1].field[0].internalid 84639357 +doctype[3].structtype[1].field[0].type 10012 +doctype[4].name "foobar" +doctype[4].idx 10024 +doctype[4].inherits[0].idx 10000 +doctype[4].inherits[1].idx 10018 +doctype[4].inherits[2].idx 10021 +doctype[4].contentstruct 10025 +doctype[4].structtype[0].idx 10025 +doctype[4].structtype[0].name "foobar.header" +doctype[4].structtype[1].idx 10026 +doctype[4].structtype[1].name "mystructinfoobar" +doctype[4].structtype[1].field[0].name "f3" +doctype[4].structtype[1].field[0].internalid 63940691 +doctype[4].structtype[1].field[0].type 10012 diff --git a/config-model/src/test/derived/declstruct/foo.sd b/config-model/src/test/derived/declstruct/foo.sd new file mode 100644 index 00000000000..6470671f437 --- /dev/null +++ b/config-model/src/test/derived/declstruct/foo.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search foo { + document foo inherits common { + struct mystructinfoo { + field f1 type string { } + } + } +} diff --git a/config-model/src/test/derived/declstruct/foobar.sd b/config-model/src/test/derived/declstruct/foobar.sd new file mode 100644 index 00000000000..cc05444249b --- /dev/null +++ b/config-model/src/test/derived/declstruct/foobar.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search foobar { + document foobar inherits foo, bar { + struct mystructinfoobar { + field f3 type string { } + } + } +} diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index f4234aee087..a6530d49507 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -1,37 +1,55 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 461724009 -datatype[].structtype[].name "emptydefault.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "one" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "two" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1663995626 -datatype[].documenttype[].name "emptydefault" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 461724009 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "one" -datatype[].documenttype[].fieldsets{[document]}.fields[] "two" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "emptydefault" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "one" +doctype[].fieldsets{[document]}.fields[] "two" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "emptydefault.header" +doctype[].structtype[].field[].name "one" +doctype[].structtype[].field[].internalid 997119011 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "two" +doctype[].structtype[].field[].internalid 2054688289 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index dad69375887..fafea7dd4bc 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -1,33 +1,51 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -531633022 -datatype[].structtype[].name "id.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "uri" -datatype[].structtype[].field[].datatype 10 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 3225629 -datatype[].documenttype[].name "id" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -531633022 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "uri" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "id" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "uri" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "id.header" +doctype[].structtype[].field[].name "uri" +doctype[].structtype[].field[].internalid 933242992 +doctype[].structtype[].field[].type 10014 diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index 8ba249ed751..7f6b9150e07 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -1,46 +1,64 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -555640823 -datatype[].structtype[].name "indexswitches.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "title" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "descr" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "source_src" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "source" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -753375626 -datatype[].documenttype[].name "indexswitches" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -555640823 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{default}.fields[] "descr" -datatype[].documenttype[].fieldsets{default}.fields[] "title" -datatype[].documenttype[].fieldsets{[document]}.fields[] "descr" -datatype[].documenttype[].fieldsets{[document]}.fields[] "source_src" -datatype[].documenttype[].fieldsets{[document]}.fields[] "title" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "indexswitches" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{default}.fields[] "descr" +doctype[].fieldsets{default}.fields[] "title" +doctype[].fieldsets{[document]}.fields[] "descr" +doctype[].fieldsets{[document]}.fields[] "source_src" +doctype[].fieldsets{[document]}.fields[] "title" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "indexswitches.header" +doctype[].structtype[].field[].name "title" +doctype[].structtype[].field[].internalid 567626448 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "descr" +doctype[].structtype[].field[].internalid 1499258718 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "source_src" +doctype[].structtype[].field[].internalid 563732473 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "source" +doctype[].structtype[].field[].internalid 1327368301 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/inheritance/documentmanager.cfg b/config-model/src/test/derived/inheritance/documentmanager.cfg index 4a25f8c3a64..321bf9979cd 100644 --- a/config-model/src/test/derived/inheritance/documentmanager.cfg +++ b/config-model/src/test/derived/inheritance/documentmanager.cfg @@ -4,41 +4,41 @@ doctype[0].name "document" doctype[0].idx 10000 doctype[0].contentstruct 10001 doctype[0].primitivetype[0].idx 10002 -doctype[0].primitivetype[0].name "byte" +doctype[0].primitivetype[0].name "bool" doctype[0].primitivetype[1].idx 10003 -doctype[0].primitivetype[1].name "int" +doctype[0].primitivetype[1].name "byte" doctype[0].primitivetype[2].idx 10004 -doctype[0].primitivetype[2].name "long" +doctype[0].primitivetype[2].name "double" doctype[0].primitivetype[3].idx 10005 -doctype[0].primitivetype[3].name "string" +doctype[0].primitivetype[3].name "float" doctype[0].primitivetype[4].idx 10006 -doctype[0].primitivetype[4].name "raw" -doctype[0].primitivetype[5].idx 10008 -doctype[0].primitivetype[5].name "float" -doctype[0].primitivetype[6].idx 10009 -doctype[0].primitivetype[6].name "double" -doctype[0].primitivetype[7].idx 10011 -doctype[0].primitivetype[7].name "uri" -doctype[0].primitivetype[8].idx 10012 -doctype[0].primitivetype[8].name "predicate" -doctype[0].primitivetype[9].idx 10013 -doctype[0].primitivetype[9].name "bool" +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" doctype[0].primitivetype[10].idx 10014 -doctype[0].primitivetype[10].name "float16" -doctype[0].wsettype[0].idx 10007 -doctype[0].wsettype[0].elementtype 10005 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 doctype[0].wsettype[0].createifnonexistent true doctype[0].wsettype[0].removeifzero true doctype[0].structtype[0].idx 10001 doctype[0].structtype[0].name "document.header" -doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].idx 10009 doctype[0].structtype[1].name "position" doctype[0].structtype[1].field[0].name "x" doctype[0].structtype[1].field[0].internalid 914677694 -doctype[0].structtype[1].field[0].type 10003 +doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 -doctype[0].structtype[1].field[1].type 10003 +doctype[0].structtype[1].field[1].type 10007 doctype[1].name "grandparent" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 @@ -49,10 +49,10 @@ doctype[1].structtype[0].idx 10016 doctype[1].structtype[0].name "grandparent.header" doctype[1].structtype[0].field[0].name "onlygrandparent" doctype[1].structtype[0].field[0].internalid 1456982690 -doctype[1].structtype[0].field[0].type 10003 +doctype[1].structtype[0].field[0].type 10007 doctype[1].structtype[0].field[1].name "overridden" doctype[1].structtype[0].field[1].internalid 1314355415 -doctype[1].structtype[0].field[1].type 10003 +doctype[1].structtype[0].field[1].type 10007 doctype[2].name "mother" doctype[2].idx 10017 doctype[2].inherits[0].idx 10015 @@ -65,10 +65,10 @@ doctype[2].structtype[0].idx 10018 doctype[2].structtype[0].name "mother.header" doctype[2].structtype[0].field[0].name "onlymother" doctype[2].structtype[0].field[0].internalid 1390999339 -doctype[2].structtype[0].field[0].type 10005 +doctype[2].structtype[0].field[0].type 10012 doctype[2].structtype[0].field[1].name "overridden" doctype[2].structtype[0].field[1].internalid 1314355415 -doctype[2].structtype[0].field[1].type 10003 +doctype[2].structtype[0].field[1].type 10007 doctype[3].name "father" doctype[3].idx 10019 doctype[3].inherits[0].idx 10015 @@ -81,10 +81,10 @@ doctype[3].structtype[0].idx 10020 doctype[3].structtype[0].name "father.header" doctype[3].structtype[0].field[0].name "onlyfather" doctype[3].structtype[0].field[0].internalid 1083094308 -doctype[3].structtype[0].field[0].type 10005 +doctype[3].structtype[0].field[0].type 10012 doctype[3].structtype[0].field[1].name "overridden" doctype[3].structtype[0].field[1].internalid 1314355415 -doctype[3].structtype[0].field[1].type 10003 +doctype[3].structtype[0].field[1].type 10007 doctype[4].name "child" doctype[4].idx 10021 doctype[4].inherits[0].idx 10000 @@ -100,7 +100,7 @@ doctype[4].structtype[0].idx 10022 doctype[4].structtype[0].name "child.header" doctype[4].structtype[0].field[0].name "onlychild" doctype[4].structtype[0].field[0].internalid 1737375598 -doctype[4].structtype[0].field[0].type 10005 +doctype[4].structtype[0].field[0].type 10012 doctype[4].structtype[0].field[1].name "overridden" doctype[4].structtype[0].field[1].internalid 1314355415 -doctype[4].structtype[0].field[1].type 10003 +doctype[4].structtype[0].field[1].type 10007 diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index cc76fe939b0..f4398e59f0f 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -4,41 +4,41 @@ doctype[0].name "document" doctype[0].idx 10000 doctype[0].contentstruct 10001 doctype[0].primitivetype[0].idx 10002 -doctype[0].primitivetype[0].name "byte" +doctype[0].primitivetype[0].name "bool" doctype[0].primitivetype[1].idx 10003 -doctype[0].primitivetype[1].name "int" +doctype[0].primitivetype[1].name "byte" doctype[0].primitivetype[2].idx 10004 -doctype[0].primitivetype[2].name "long" +doctype[0].primitivetype[2].name "double" doctype[0].primitivetype[3].idx 10005 -doctype[0].primitivetype[3].name "string" +doctype[0].primitivetype[3].name "float" doctype[0].primitivetype[4].idx 10006 -doctype[0].primitivetype[4].name "raw" -doctype[0].primitivetype[5].idx 10008 -doctype[0].primitivetype[5].name "float" -doctype[0].primitivetype[6].idx 10009 -doctype[0].primitivetype[6].name "double" -doctype[0].primitivetype[7].idx 10011 -doctype[0].primitivetype[7].name "uri" -doctype[0].primitivetype[8].idx 10012 -doctype[0].primitivetype[8].name "predicate" -doctype[0].primitivetype[9].idx 10013 -doctype[0].primitivetype[9].name "bool" +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" doctype[0].primitivetype[10].idx 10014 -doctype[0].primitivetype[10].name "float16" -doctype[0].wsettype[0].idx 10007 -doctype[0].wsettype[0].elementtype 10005 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 doctype[0].wsettype[0].createifnonexistent true doctype[0].wsettype[0].removeifzero true doctype[0].structtype[0].idx 10001 doctype[0].structtype[0].name "document.header" -doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].idx 10009 doctype[0].structtype[1].name "position" doctype[0].structtype[1].field[0].name "x" doctype[0].structtype[1].field[0].internalid 914677694 -doctype[0].structtype[1].field[0].type 10003 +doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 -doctype[0].structtype[1].field[1].type 10003 +doctype[0].structtype[1].field[1].type 10007 doctype[1].name "grandparent" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 @@ -49,7 +49,7 @@ doctype[1].structtype[1].idx 10017 doctype[1].structtype[1].name "grandparent_struct" doctype[1].structtype[1].field[0].name "grandparent_field" doctype[1].structtype[1].field[0].internalid 18801796 -doctype[1].structtype[1].field[0].type 10005 +doctype[1].structtype[1].field[0].type 10012 doctype[2].name "parent" doctype[2].idx 10018 doctype[2].inherits[0].idx 10015 diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index 3c7280094be..1363453017f 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -4,41 +4,41 @@ doctype[0].name "document" doctype[0].idx 10000 doctype[0].contentstruct 10001 doctype[0].primitivetype[0].idx 10002 -doctype[0].primitivetype[0].name "byte" +doctype[0].primitivetype[0].name "bool" doctype[0].primitivetype[1].idx 10003 -doctype[0].primitivetype[1].name "int" +doctype[0].primitivetype[1].name "byte" doctype[0].primitivetype[2].idx 10004 -doctype[0].primitivetype[2].name "long" +doctype[0].primitivetype[2].name "double" doctype[0].primitivetype[3].idx 10005 -doctype[0].primitivetype[3].name "string" +doctype[0].primitivetype[3].name "float" doctype[0].primitivetype[4].idx 10006 -doctype[0].primitivetype[4].name "raw" -doctype[0].primitivetype[5].idx 10008 -doctype[0].primitivetype[5].name "float" -doctype[0].primitivetype[6].idx 10009 -doctype[0].primitivetype[6].name "double" -doctype[0].primitivetype[7].idx 10011 -doctype[0].primitivetype[7].name "uri" -doctype[0].primitivetype[8].idx 10012 -doctype[0].primitivetype[8].name "predicate" -doctype[0].primitivetype[9].idx 10013 -doctype[0].primitivetype[9].name "bool" +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" doctype[0].primitivetype[10].idx 10014 -doctype[0].primitivetype[10].name "float16" -doctype[0].wsettype[0].idx 10007 -doctype[0].wsettype[0].elementtype 10005 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 doctype[0].wsettype[0].createifnonexistent true doctype[0].wsettype[0].removeifzero true doctype[0].structtype[0].idx 10001 doctype[0].structtype[0].name "document.header" -doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].idx 10009 doctype[0].structtype[1].name "position" doctype[0].structtype[1].field[0].name "x" doctype[0].structtype[1].field[0].internalid 914677694 -doctype[0].structtype[1].field[0].type 10003 +doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 -doctype[0].structtype[1].field[1].type 10003 +doctype[0].structtype[1].field[1].type 10007 doctype[1].name "parent" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 @@ -48,15 +48,15 @@ doctype[1].structtype[0].idx 10016 doctype[1].structtype[0].name "parent.header" doctype[1].structtype[0].field[0].name "weight_src" doctype[1].structtype[0].field[0].internalid 1225660233 -doctype[1].structtype[0].field[0].type 10008 +doctype[1].structtype[0].field[0].type 10005 doctype[1].structtype[0].field[1].name "weight" doctype[1].structtype[0].field[1].internalid 1001392207 -doctype[1].structtype[0].field[1].type 10008 +doctype[1].structtype[0].field[1].type 10005 doctype[1].structtype[1].idx 10017 doctype[1].structtype[1].name "parent_struct" doctype[1].structtype[1].field[0].name "parent_field" doctype[1].structtype[1].field[0].internalid 933533022 -doctype[1].structtype[1].field[0].type 10005 +doctype[1].structtype[1].field[0].type 10012 doctype[2].name "child" doctype[2].idx 10018 doctype[2].inherits[0].idx 10000 diff --git a/config-model/src/test/derived/mail/documentmanager.cfg b/config-model/src/test/derived/mail/documentmanager.cfg index b6fdbe8f210..e69de29bb2d 100644 --- a/config-model/src/test/derived/mail/documentmanager.cfg +++ b/config-model/src/test/derived/mail/documentmanager.cfg @@ -1,105 +0,0 @@ -enablecompression false -usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].id -88808602 -datatype[].structtype[].name "mail.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "URI" -datatype[].structtype[].field[].datatype 10 -datatype[].structtype[].field[].name "mailid" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "date" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].name "from" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "replyto" -datatype[].structtype[].field[].datatype 12 -datatype[].structtype[].field[].name "to" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "cc" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "bcc" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "subject" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "snippet" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "body" -datatype[].structtype[].field[].datatype 12 -datatype[].structtype[].field[].name "attachmentcount" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].name "attachmentnames" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "attachmenttypes" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "attachmentlanguages" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "attachmentcontent" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "attachments" -datatype[].structtype[].field[].datatype -1206550296 -datatype[].id -1206550296 -datatype[].arraytype[].datatype 12 -datatype[].id -1081574983 -datatype[].documenttype[].name "mail" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -88808602 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{sender}.fields[] "from" -datatype[].documenttype[].fieldsets{address}.fields[] "cc" -datatype[].documenttype[].fieldsets{address}.fields[] "from" -datatype[].documenttype[].fieldsets{address}.fields[] "to" -datatype[].documenttype[].fieldsets{header}.fields[] "cc" -datatype[].documenttype[].fieldsets{header}.fields[] "from" -datatype[].documenttype[].fieldsets{header}.fields[] "subject" -datatype[].documenttype[].fieldsets{header}.fields[] "to" -datatype[].documenttype[].fieldsets{default}.fields[] "body" -datatype[].documenttype[].fieldsets{default}.fields[] "cc" -datatype[].documenttype[].fieldsets{default}.fields[] "from" -datatype[].documenttype[].fieldsets{default}.fields[] "subject" -datatype[].documenttype[].fieldsets{default}.fields[] "to" -datatype[].documenttype[].fieldsets{all}.fields[] "attachmentcontent" -datatype[].documenttype[].fieldsets{all}.fields[] "attachmentnames" -datatype[].documenttype[].fieldsets{all}.fields[] "attachmenttypes" -datatype[].documenttype[].fieldsets{all}.fields[] "body" -datatype[].documenttype[].fieldsets{all}.fields[] "cc" -datatype[].documenttype[].fieldsets{all}.fields[] "from" -datatype[].documenttype[].fieldsets{all}.fields[] "subject" -datatype[].documenttype[].fieldsets{all}.fields[] "to" -datatype[].documenttype[].fieldsets{recipient}.fields[] "cc" -datatype[].documenttype[].fieldsets{recipient}.fields[] "to" -datatype[].documenttype[].fieldsets{attachmentname}.fields[] "attachmentnames" -datatype[].documenttype[].fieldsets{attachmenttype}.fields[] "attachmenttypes" -datatype[].documenttype[].fieldsets{attachment}.fields[] "attachmentcontent" -datatype[].documenttype[].fieldsets{[document]}.fields[] "URI" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attachmentcontent" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attachmentcount" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attachmentlanguages" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attachmentnames" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attachments" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attachmenttypes" -datatype[].documenttype[].fieldsets{[document]}.fields[] "bcc" -datatype[].documenttype[].fieldsets{[document]}.fields[] "body" -datatype[].documenttype[].fieldsets{[document]}.fields[] "cc" -datatype[].documenttype[].fieldsets{[document]}.fields[] "date" -datatype[].documenttype[].fieldsets{[document]}.fields[] "from" -datatype[].documenttype[].fieldsets{[document]}.fields[] "mailid" -datatype[].documenttype[].fieldsets{[document]}.fields[] "replyto" -datatype[].documenttype[].fieldsets{[document]}.fields[] "subject" -datatype[].documenttype[].fieldsets{[document]}.fields[] "to" diff --git a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg new file mode 100644 index 00000000000..374d5071c31 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg @@ -0,0 +1,35 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "exact @mse4_key@" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "exact @mse4_value_sf1s@" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "type Map<string,elem>" diff --git a/config-model/src/test/derived/matchsettings_map_after/test.sd b/config-model/src/test/derived/matchsettings_map_after/test.sd new file mode 100644 index 00000000000..2f6372628c9 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_after/test.sd @@ -0,0 +1,34 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + field mse4 type map<string,elem> { + indexing: summary + struct-field key { + indexing: index + match { + exact + exact-terminator: "@mse4_key@" + } + } + struct-field value.sf1s { + indexing: index + match { + exact + exact-terminator: "@mse4_value_sf1s@" + } + } + struct-field value.sf2i { + indexing: attribute + } + } + + struct elem { + field sf1s type string {} + field sf2i type int {} + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg new file mode 100644 index 00000000000..4e87847265a --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg @@ -0,0 +1,105 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "type Map<string,string>" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "type Map<string,elem>" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "type Map<string,elem>" diff --git a/config-model/src/test/derived/matchsettings_map_def/test.sd b/config-model/src/test/derived/matchsettings_map_def/test.sd new file mode 100644 index 00000000000..7f2517c80fc --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_def/test.sd @@ -0,0 +1,50 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field sf1s type string {} + field sf2i type int {} + } + + field mss3 type map<string,string> { + indexing: index + struct-field key { + indexing: index + } + struct-field value { + indexing: index + } + } + + field mse4 type map<string,elem> { + indexing: summary + struct-field key { + indexing: index + } + struct-field value.sf1s { + indexing: index + } + struct-field value.sf2i { + indexing: attribute + } + } + + field mse5 type map<string,elem> { + indexing: summary + struct-field key { + indexing: attribute + attribute: fast-search + } + struct-field value.sf1s { + indexing: attribute + } + struct-field value.sf2i { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg new file mode 100644 index 00000000000..38ae1f78d17 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg @@ -0,0 +1,101 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "exact @mss3_key@" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "exact @mss3_value@" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "type Map<string,string>" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "exact @mse4_key@" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "exact @mse4_value_sf1s@" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "type Map<string,elem>" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "exact @mse5_key@" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "exact @mse5_value_sf1s@" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "type Map<string,elem>" diff --git a/config-model/src/test/derived/matchsettings_map_wfs/test.sd b/config-model/src/test/derived/matchsettings_map_wfs/test.sd new file mode 100644 index 00000000000..12c0c395282 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_wfs/test.sd @@ -0,0 +1,72 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field sf1s type string {} + field sf2i type int {} + } + + field mss3 type map<string,string> { + indexing: index + struct-field key { + match { + exact + exact-terminator: "@mss3_key@" + } + } + struct-field value { + match { + exact + exact-terminator: "@mss3_value@" + } + } + } + + field mse4 type map<string,elem> { + indexing: summary + struct-field key { + indexing: index + match { + exact + exact-terminator: "@mse4_key@" + } + } + struct-field value.sf1s { + indexing: index + match { + exact + exact-terminator: "@mse4_value_sf1s@" + } + } + struct-field value.sf2i { + indexing: attribute + } + } + + field mse5 type map<string,elem> { + indexing: summary + struct-field key { + indexing: attribute + attribute: fast-search + match { + exact + exact-terminator: "@mse5_key@" + } + } + struct-field value.sf1s { + indexing: attribute + match { + exact + exact-terminator: "@mse5_value_sf1s@" + } + } + struct-field value.sf2i { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg new file mode 100644 index 00000000000..0f6773d0fe8 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg @@ -0,0 +1,107 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "mss3" +indexinfo[].command[].command "type Map<string,string>" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse4.value.sf1s" +indexinfo[].command[].command "exact @elem_sf1s@" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse4.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4" +indexinfo[].command[].command "type Map<string,elem>" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "mse5.value.sf1s" +indexinfo[].command[].command "exact @elem_sf1s@" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mse5.value.sf2i" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5" +indexinfo[].command[].command "type Map<string,elem>" diff --git a/config-model/src/test/derived/matchsettings_map_wss/test.sd b/config-model/src/test/derived/matchsettings_map_wss/test.sd new file mode 100644 index 00000000000..58162ea93fc --- /dev/null +++ b/config-model/src/test/derived/matchsettings_map_wss/test.sd @@ -0,0 +1,55 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field sf1s type string { + match { + exact + exact-terminator: "@elem_sf1s@" + } + } + field sf2i type int {} + } + + field mss3 type map<string,string> { + indexing: index + struct-field key { + indexing: index + } + struct-field value { + indexing: index + } + } + + field mse4 type map<string,elem> { + indexing: summary + struct-field key { + indexing: index + } + struct-field value.sf1s { + indexing: index + } + struct-field value.sf2i { + indexing: attribute + } + } + + field mse5 type map<string,elem> { + indexing: summary + struct-field key { + indexing: attribute + attribute: fast-search + } + struct-field value.sf1s { + indexing: attribute + } + struct-field value.sf2i { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_simple_def/index-info.cfg b/config-model/src/test/derived/matchsettings_simple_def/index-info.cfg new file mode 100644 index 00000000000..fbd7b7755cc --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_def/index-info.cfg @@ -0,0 +1,33 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "type elem" diff --git a/config-model/src/test/derived/matchsettings_simple_def/test.sd b/config-model/src/test/derived/matchsettings_simple_def/test.sd new file mode 100644 index 00000000000..8d5a1733a44 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_def/test.sd @@ -0,0 +1,27 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field elem_field_1_string type string {} + field elem_field_2_int type int {} + } + + field field_1_string type string { + indexing: index + } + + field field_2_struct type elem { + indexing: summary + struct-field elem_field_1_string { + indexing: index + } + struct-field elem_field_2_int { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_simple_wfs/index-info.cfg b/config-model/src/test/derived/matchsettings_simple_wfs/index-info.cfg new file mode 100644 index 00000000000..8362870f427 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_wfs/index-info.cfg @@ -0,0 +1,31 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "exact @f1s@" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "exact @f2s@" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "type elem" diff --git a/config-model/src/test/derived/matchsettings_simple_wfs/test.sd b/config-model/src/test/derived/matchsettings_simple_wfs/test.sd new file mode 100644 index 00000000000..9c709d8167e --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_wfs/test.sd @@ -0,0 +1,35 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field elem_field_1_string type string {} + field elem_field_2_int type int {} + } + + field field_1_string type string { + indexing: index + match { + exact + exact-terminator: "@f1s@" + } + } + + field field_2_struct type elem { + indexing: summary + struct-field elem_field_1_string { + indexing: index + match { + exact + exact-terminator: "@f2s@" + } + } + struct-field elem_field_2_int { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_simple_wss/index-info.cfg b/config-model/src/test/derived/matchsettings_simple_wss/index-info.cfg new file mode 100644 index 00000000000..dda78b04f9b --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_wss/index-info.cfg @@ -0,0 +1,31 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "exact @f1s@" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "exact @ef1@" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "type elem" diff --git a/config-model/src/test/derived/matchsettings_simple_wss/test.sd b/config-model/src/test/derived/matchsettings_simple_wss/test.sd new file mode 100644 index 00000000000..451d9ad06d8 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_wss/test.sd @@ -0,0 +1,36 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field elem_field_1_string type string { + match { + exact + exact-terminator: "@ef1@" + } + } + field elem_field_2_int type int {} + } + + field field_1_string type string { + indexing: index + match { + exact + exact-terminator: "@f1s@" + } + } + + field field_2_struct type elem { + indexing: summary + struct-field elem_field_1_string { + indexing: index + } + struct-field elem_field_2_int { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/matchsettings_simple_wss_wfs/index-info.cfg b/config-model/src/test/derived/matchsettings_simple_wss_wfs/index-info.cfg new file mode 100644 index 00000000000..8362870f427 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_wss_wfs/index-info.cfg @@ -0,0 +1,31 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_1_string" +indexinfo[].command[].command "exact @f1s@" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "field_2_struct.elem_field_1_string" +indexinfo[].command[].command "exact @f2s@" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "field_2_struct.elem_field_2_int" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field_2_struct" +indexinfo[].command[].command "type elem" diff --git a/config-model/src/test/derived/matchsettings_simple_wss_wfs/test.sd b/config-model/src/test/derived/matchsettings_simple_wss_wfs/test.sd new file mode 100644 index 00000000000..f3967e6ad80 --- /dev/null +++ b/config-model/src/test/derived/matchsettings_simple_wss_wfs/test.sd @@ -0,0 +1,40 @@ + +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +search test { + document test { + + struct elem { + field elem_field_1_string type string { + match { + exact + exact-terminator: "@ef1@" + } + } + field elem_field_2_int type int {} + } + + field field_1_string type string { + indexing: index + match { + exact + exact-terminator: "@f1s@" + } + } + + field field_2_struct type elem { + indexing: summary + struct-field elem_field_1_string { + indexing: index + match { + exact + exact-terminator: "@f2s@" + } + } + struct-field elem_field_2_int { + indexing: attribute + } + } + + } +} diff --git a/config-model/src/test/derived/multi_struct/ad.sd b/config-model/src/test/derived/multi_struct/ad.sd new file mode 100644 index 00000000000..a14cd6dc023 --- /dev/null +++ b/config-model/src/test/derived/multi_struct/ad.sd @@ -0,0 +1,7 @@ +search ad { + document ad { + field e type string { + indexing: attribute | summary + } + } +} diff --git a/config-model/src/test/derived/multi_struct/documentmanager.cfg b/config-model/src/test/derived/multi_struct/documentmanager.cfg new file mode 100644 index 00000000000..58775331d58 --- /dev/null +++ b/config-model/src/test/derived/multi_struct/documentmanager.cfg @@ -0,0 +1,109 @@ +enablecompression false +usev8geopositions false +datatype[].id 1381038251 +datatype[].structtype[].name "position" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "x" +datatype[].structtype[].field[].datatype 0 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "y" +datatype[].structtype[].field[].datatype 0 +datatype[].structtype[].field[].detailedtype "" +datatype[].id 959075962 +datatype[].structtype[].name "ad.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "e" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].id 2987301 +datatype[].documenttype[].name "ad" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct 959075962 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "e" +datatype[].id -420192670 +datatype[].maptype[].keytype 2 +datatype[].maptype[].valtype -2092985853 +datatype[].id -2092985853 +datatype[].structtype[].name "mystruct" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "zero" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "one" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "two" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "three" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].id -107300050 +datatype[].structtype[].name "product.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "more_stuff" +datatype[].structtype[].field[].datatype -420192670 +datatype[].structtype[].field[].detailedtype "" +datatype[].id -1051831567 +datatype[].documenttype[].name "product" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct -107300050 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "more_stuff" +datatype[].id 371492103 +datatype[].structtype[].name "shop.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "some_stuff" +datatype[].structtype[].field[].datatype -420192670 +datatype[].structtype[].field[].detailedtype "" +datatype[].id -903152840 +datatype[].documenttype[].name "shop" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct 371492103 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "some_stuff" +datatype[].id 1601213394 +datatype[].structtype[].name "user.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "a" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].id -836031795 +datatype[].documenttype[].name "user" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct 1601213394 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "a" diff --git a/config-model/src/test/derived/multi_struct/documenttypes.cfg b/config-model/src/test/derived/multi_struct/documenttypes.cfg new file mode 100644 index 00000000000..c358b2854ee --- /dev/null +++ b/config-model/src/test/derived/multi_struct/documenttypes.cfg @@ -0,0 +1,194 @@ +enablecompression false +usev8geopositions false +documenttype[].id 2987301 +documenttype[].name "ad" +documenttype[].version 0 +documenttype[].headerstruct 959075962 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id 959075962 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "ad.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "e" +documenttype[].datatype[].sstruct.field[].id 970377814 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[document]}.fields[] "e" +documenttype[].id -1051831567 +documenttype[].name "product" +documenttype[].version 0 +documenttype[].headerstruct -107300050 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id -2092985853 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "mystruct" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "zero" +documenttype[].datatype[].sstruct.field[].id 2128579715 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].sstruct.field[].name "one" +documenttype[].datatype[].sstruct.field[].id 997119011 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].sstruct.field[].name "two" +documenttype[].datatype[].sstruct.field[].id 2054688289 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].sstruct.field[].name "three" +documenttype[].datatype[].sstruct.field[].id 814368361 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].id -420192670 +documenttype[].datatype[].type MAP +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 2 +documenttype[].datatype[].map.value.id -2092985853 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].id -107300050 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "product.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "more_stuff" +documenttype[].datatype[].sstruct.field[].id 278342855 +documenttype[].datatype[].sstruct.field[].datatype -420192670 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[document]}.fields[] "more_stuff" +documenttype[].id -903152840 +documenttype[].name "shop" +documenttype[].version 0 +documenttype[].headerstruct 371492103 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id -2092985853 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "mystruct" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "one" +documenttype[].datatype[].sstruct.field[].id 997119011 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].sstruct.field[].name "two" +documenttype[].datatype[].sstruct.field[].id 2054688289 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].sstruct.field[].name "three" +documenttype[].datatype[].sstruct.field[].id 814368361 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].datatype[].id -420192670 +documenttype[].datatype[].type MAP +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 2 +documenttype[].datatype[].map.value.id -2092985853 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].id 371492103 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "shop.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "some_stuff" +documenttype[].datatype[].sstruct.field[].id 1543312381 +documenttype[].datatype[].sstruct.field[].datatype -420192670 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[document]}.fields[] "some_stuff" +documenttype[].id -836031795 +documenttype[].name "user" +documenttype[].version 0 +documenttype[].headerstruct 1601213394 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id 1601213394 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "user.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "a" +documenttype[].datatype[].sstruct.field[].id 493339625 +documenttype[].datatype[].sstruct.field[].datatype 2 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[document]}.fields[] "a" diff --git a/config-model/src/test/derived/multi_struct/product.sd b/config-model/src/test/derived/multi_struct/product.sd new file mode 100644 index 00000000000..cba611cd794 --- /dev/null +++ b/config-model/src/test/derived/multi_struct/product.sd @@ -0,0 +1,13 @@ +search product { + document product { + struct mystruct { + field zero type string {} + field one type string {} + field two type string {} + field three type string {} + } + field more_stuff type map<string, mystruct> { + indexing: summary + } + } +} diff --git a/config-model/src/test/derived/multi_struct/shop.sd b/config-model/src/test/derived/multi_struct/shop.sd new file mode 100644 index 00000000000..b5c06cc493f --- /dev/null +++ b/config-model/src/test/derived/multi_struct/shop.sd @@ -0,0 +1,12 @@ +search shop { + document shop { + struct mystruct { + field one type string {} + field two type string {} + field three type string {} + } + field some_stuff type map<string, mystruct> { + indexing: summary + } + } +} diff --git a/config-model/src/test/derived/multi_struct/user.sd b/config-model/src/test/derived/multi_struct/user.sd new file mode 100644 index 00000000000..ec23e9f88ea --- /dev/null +++ b/config-model/src/test/derived/multi_struct/user.sd @@ -0,0 +1,7 @@ +search user { + document user { + field a type string { + indexing: summary + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/derived/namecollision/documentmanager.cfg b/config-model/src/test/derived/namecollision/documentmanager.cfg index 99da89f4fbf..73730595574 100644 --- a/config-model/src/test/derived/namecollision/documentmanager.cfg +++ b/config-model/src/test/derived/namecollision/documentmanager.cfg @@ -1,56 +1,61 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -379118517 -datatype[].structtype[].name "collision.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id 1557022836 -datatype[].documenttype[].name "collision" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -379118517 -datatype[].documenttype[].bodystruct 0 -datatype[].id 1557022836 -datatype[].structtype[].name "collision" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id -1730522993 -datatype[].arraytype[].datatype 1557022836 -datatype[].id -1270379114 -datatype[].structtype[].name "collisionstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "structarray" -datatype[].structtype[].field[].datatype -1730522993 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1723079287 -datatype[].documenttype[].name "collisionstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -1270379114 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[]}.fields[] "structarray" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "collision" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "collision.header" +doctype[].name "collisionstruct" +doctype[].idx 10017 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10018 +doctype[].fieldsets{[document]}.fields[] "structarray" +doctype[].arraytype[].idx 10019 +doctype[].arraytype[].elementtype 10020 +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "collision" +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "collisionstruct.header" +doctype[].structtype[].field[].name "structarray" +doctype[].structtype[].field[].internalid 828511543 +doctype[].structtype[].field[].type 10019 diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index e37ea304b18..be048ee2eba 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -1,49 +1,67 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -739138930 -datatype[].structtype[].name "prefixexactattribute.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "indexfield0" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "attributefield1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "attributefield2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "indexfield1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "indexfield2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1812793455 -datatype[].documenttype[].name "prefixexactattribute" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -739138930 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "attributefield1" -datatype[].documenttype[].fieldsets{[document]}.fields[] "attributefield2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "indexfield0" -datatype[].documenttype[].fieldsets{[document]}.fields[] "indexfield1" -datatype[].documenttype[].fieldsets{[document]}.fields[] "indexfield2" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "prefixexactattribute" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "attributefield1" +doctype[].fieldsets{[document]}.fields[] "attributefield2" +doctype[].fieldsets{[document]}.fields[] "indexfield0" +doctype[].fieldsets{[document]}.fields[] "indexfield1" +doctype[].fieldsets{[document]}.fields[] "indexfield2" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "prefixexactattribute.header" +doctype[].structtype[].field[].name "indexfield0" +doctype[].structtype[].field[].internalid 645342977 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "attributefield1" +doctype[].structtype[].field[].internalid 76109497 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "attributefield2" +doctype[].structtype[].field[].internalid 1802174375 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "indexfield1" +doctype[].structtype[].field[].internalid 428289584 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "indexfield2" +doctype[].structtype[].field[].internalid 901251686 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index dc02052c509..33d41084651 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -1,48 +1,66 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -471393776 -datatype[].structtype[].name "ranktypes.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "title" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "descr" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "keywords" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "identity" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "identity_literal" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -883421617 -datatype[].documenttype[].name "ranktypes" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -471393776 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "descr" -datatype[].documenttype[].fieldsets{[document]}.fields[] "identity" -datatype[].documenttype[].fieldsets{[document]}.fields[] "keywords" -datatype[].documenttype[].fieldsets{[document]}.fields[] "title" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "ranktypes" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "descr" +doctype[].fieldsets{[document]}.fields[] "identity" +doctype[].fieldsets{[document]}.fields[] "keywords" +doctype[].fieldsets{[document]}.fields[] "title" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "ranktypes.header" +doctype[].structtype[].field[].name "title" +doctype[].structtype[].field[].internalid 567626448 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "descr" +doctype[].structtype[].field[].internalid 1499258718 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "keywords" +doctype[].structtype[].field[].internalid 1813672906 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "identity" +doctype[].structtype[].field[].internalid 92760117 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "identity_literal" +doctype[].structtype[].field[].internalid 459073620 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/reference_from_several/bar.sd b/config-model/src/test/derived/reference_from_several/bar.sd new file mode 100644 index 00000000000..68e1b22da3a --- /dev/null +++ b/config-model/src/test/derived/reference_from_several/bar.sd @@ -0,0 +1,15 @@ +schema bar { + document bar { + field bpref type reference<parent> { + indexing: attribute + } + field bartitle type string { + indexing: attribute + } + } + import field bpref.x as barsximp {} + document-summary other { + summary bartitle type string {} + summary barsximp type int {} + } +} diff --git a/config-model/src/test/derived/reference_from_several/documentmanager.cfg b/config-model/src/test/derived/reference_from_several/documentmanager.cfg new file mode 100644 index 00000000000..88dee077539 --- /dev/null +++ b/config-model/src/test/derived/reference_from_several/documentmanager.cfg @@ -0,0 +1,81 @@ +enablecompression false +usev8geopositions false +datatype[].id 1381038251 +datatype[].structtype[].name "position" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "x" +datatype[].structtype[].field[].datatype 0 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "y" +datatype[].structtype[].field[].datatype 0 +datatype[].structtype[].field[].detailedtype "" +datatype[].id 427398467 +datatype[].referencetype[].target_type_id 1175161836 +datatype[].id -2026908534 +datatype[].structtype[].name "bar.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "bpref" +datatype[].structtype[].field[].datatype 427398467 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "bartitle" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].id 93505813 +datatype[].documenttype[].name "bar" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct -2026908534 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "bartitle" +datatype[].documenttype[].fieldsets{[document]}.fields[] "bpref" +datatype[].documenttype[].importedfield[].name "barsximp" +datatype[].id -308552393 +datatype[].structtype[].name "foo.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "myref" +datatype[].structtype[].field[].datatype 427398467 +datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "foo" +datatype[].structtype[].field[].datatype 2 +datatype[].structtype[].field[].detailedtype "" +datatype[].id 97614088 +datatype[].documenttype[].name "foo" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct -308552393 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "foo" +datatype[].documenttype[].fieldsets{[document]}.fields[] "myref" +datatype[].documenttype[].importedfield[].name "myx" +datatype[].id 836075987 +datatype[].structtype[].name "parent.header" +datatype[].structtype[].version 0 +datatype[].structtype[].compresstype NONE +datatype[].structtype[].compresslevel 0 +datatype[].structtype[].compressthreshold 95 +datatype[].structtype[].compressminsize 800 +datatype[].structtype[].field[].name "x" +datatype[].structtype[].field[].datatype 0 +datatype[].structtype[].field[].detailedtype "" +datatype[].id 1175161836 +datatype[].documenttype[].name "parent" +datatype[].documenttype[].version 0 +datatype[].documenttype[].inherits[].name "document" +datatype[].documenttype[].inherits[].version 0 +datatype[].documenttype[].headerstruct 836075987 +datatype[].documenttype[].bodystruct 0 +datatype[].documenttype[].fieldsets{[document]}.fields[] "x" diff --git a/config-model/src/test/derived/reference_from_several/foo.sd b/config-model/src/test/derived/reference_from_several/foo.sd new file mode 100644 index 00000000000..bfc4eca28c2 --- /dev/null +++ b/config-model/src/test/derived/reference_from_several/foo.sd @@ -0,0 +1,15 @@ +schema foo { + document foo { + field myref type reference<parent> { + indexing: attribute + } + field foo type string { + indexing: attribute | summary + } + } + import field myref.x as myx {} + document-summary small { + summary myx type int {} + summary foo type string {} + } +} diff --git a/config-model/src/test/derived/reference_from_several/parent.sd b/config-model/src/test/derived/reference_from_several/parent.sd new file mode 100644 index 00000000000..4b19c46ddec --- /dev/null +++ b/config-model/src/test/derived/reference_from_several/parent.sd @@ -0,0 +1,7 @@ +schema parent { + document parent { + field x type int { + indexing: attribute + } + } +} diff --git a/config-model/src/test/derived/schemainheritance/documentmanager.cfg b/config-model/src/test/derived/schemainheritance/documentmanager.cfg index ec13eae92ff..12c29667079 100644 --- a/config-model/src/test/derived/schemainheritance/documentmanager.cfg +++ b/config-model/src/test/derived/schemainheritance/documentmanager.cfg @@ -1,99 +1,100 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 990242616 -datatype[].structtype[].name "importedschema.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "importedfield1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "importedfield2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 167001639 -datatype[].documenttype[].name "importedschema" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 990242616 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[]}.fields[] "importedfield1" -datatype[].documenttype[].fieldsets{[]}.fields[] "importedfield2" -datatype[].id -1266134872 -datatype[].referencetype[].target_type_id 167001639 -datatype[].id 836075987 -datatype[].structtype[].name "parent.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "pf1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "importedschema_ref" -datatype[].structtype[].field[].datatype -1266134872 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "parent_field" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1175161836 -datatype[].documenttype[].name "parent" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 836075987 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{parent_set}.fields[] "pf1" -datatype[].documenttype[].fieldsets{[]}.fields[] "importedschema_ref" -datatype[].documenttype[].fieldsets{[]}.fields[] "pf1" -datatype[].documenttype[].importedfield[].name "parent_imported" -datatype[].id 81425825 -datatype[].structtype[].name "child.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "cf1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "parent_field" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "child_field" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 746267614 -datatype[].documenttype[].name "child" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].inherits[].name "parent" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{parent_set}.fields[] "pf1" -datatype[].documenttype[].fieldsets{child_set}.fields[] "cf1" -datatype[].documenttype[].fieldsets{child_set}.fields[] "pf1" -datatype[].documenttype[].fieldsets{[]}.fields[] "cf1" -datatype[].documenttype[].fieldsets{[]}.fields[] "importedschema_ref" -datatype[].documenttype[].fieldsets{[]}.fields[] "pf1" -datatype[].documenttype[].importedfield[].name "parent_imported" -datatype[].documenttype[].importedfield[].name "child_imported" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "importedschema" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "importedfield1" +doctype[].fieldsets{[document]}.fields[] "importedfield2" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "importedschema.header" +doctype[].structtype[].field[].name "importedfield1" +doctype[].structtype[].field[].internalid 847030245 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "importedfield2" +doctype[].structtype[].field[].internalid 1763348623 +doctype[].structtype[].field[].type 10012 +doctype[].name "parent" +doctype[].idx 10017 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10018 +doctype[].fieldsets{parent_set}.fields[] "pf1" +doctype[].fieldsets{[document]}.fields[] "importedschema_ref" +doctype[].fieldsets{[document]}.fields[] "pf1" +doctype[].importedfield[].name "parent_imported" +doctype[].documentref[].idx 10019 +doctype[].documentref[].targettype 10015 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "pf1" +doctype[].structtype[].field[].internalid 1474272447 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "importedschema_ref" +doctype[].structtype[].field[].internalid 243883444 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "parent_field" +doctype[].structtype[].field[].internalid 933533022 +doctype[].structtype[].field[].type 10012 +doctype[].name "child" +doctype[].idx 10020 +doctype[].inherits[].idx 10000 +doctype[].inherits[].idx 10017 +doctype[].contentstruct 10021 +doctype[].fieldsets{parent_set}.fields[] "pf1" +doctype[].fieldsets{child_set}.fields[] "cf1" +doctype[].fieldsets{child_set}.fields[] "pf1" +doctype[].fieldsets{[document]}.fields[] "cf1" +doctype[].fieldsets{[document]}.fields[] "importedschema_ref" +doctype[].fieldsets{[document]}.fields[] "pf1" +doctype[].importedfield[].name "parent_imported" +doctype[].importedfield[].name "child_imported" +doctype[].structtype[].idx 10021 +doctype[].structtype[].name "child.header" +doctype[].structtype[].field[].name "cf1" +doctype[].structtype[].field[].internalid 142606489 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "parent_field" +doctype[].structtype[].field[].internalid 933533022 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "child_field" +doctype[].structtype[].field[].internalid 1373649209 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index 5d44943535f..2bb7078a37d 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -1,4 +1,36 @@ -attribute[].name "syntaxcheck" +attribute[].name "syntaxcheck1a" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "en_US" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck2a" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.type BTREE @@ -11,8 +43,136 @@ attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false +attribute[].sortfunction RAW +attribute[].sortstrength SECONDARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck3a" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true attribute[].sortfunction LOWERCASE +attribute[].sortstrength TERTIARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck4a" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending false +attribute[].sortfunction UCA +attribute[].sortstrength QUATERNARY +attribute[].sortlocale "en_US" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck5a" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction RAW attribute[].sortstrength IDENTICAL +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck1b" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY attribute[].sortlocale "en_US" attribute[].enablebitvectors false attribute[].enableonlybitvector false @@ -30,7 +190,7 @@ attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true -attribute[].name "syntaxcheck2" +attribute[].name "syntaxcheck2b" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.type BTREE @@ -43,8 +203,72 @@ attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false +attribute[].sortfunction RAW +attribute[].sortstrength SECONDARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck3b" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true attribute[].sortfunction LOWERCASE -attribute[].sortstrength IDENTICAL +attribute[].sortstrength TERTIARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck4b" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending false +attribute[].sortfunction UCA +attribute[].sortstrength QUATERNARY attribute[].sortlocale "en_US" attribute[].enablebitvectors false attribute[].enableonlybitvector false @@ -62,6 +286,38 @@ attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true +attribute[].name "syntaxcheck5b" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction RAW +attribute[].sortstrength IDENTICAL +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE diff --git a/config-model/src/test/derived/sorting/sorting.sd b/config-model/src/test/derived/sorting/sorting.sd index e1b9a8af2f1..a08b00ed9ac 100644 --- a/config-model/src/test/derived/sorting/sorting.sd +++ b/config-model/src/test/derived/sorting/sorting.sd @@ -3,42 +3,107 @@ search sorting { document sorting { - field syntaxcheck type string { + field syntaxcheck1a type string { indexing: attribute attribute { sorting: ascending - sorting: descending sorting: function: uca - sorting: function: raw - sorting: function: lowercase sorting: strength: primary + sorting: locale: en_US + } + } + field syntaxcheck2a type string { + indexing: attribute + attribute { + sorting: descending + sorting: function: raw sorting: strength: secondary + } + } + field syntaxcheck3a type string { + indexing: attribute + attribute { + sorting: ascending + sorting: function: lowercase sorting: strength: tertiary + } + } + field syntaxcheck4a type string { + indexing: attribute + attribute { + sorting: descending + sorting: function: uca sorting: strength: quaternary - sorting: strength: identical sorting: locale: en_US } } + field syntaxcheck5a type string { + indexing: attribute + attribute { + sorting: ascending + sorting: function: raw + sorting: strength: identical + } + } - field syntaxcheck2 type string { + field syntaxcheck1b type string { indexing: attribute attribute { sorting { ascending - descending function: uca - function: raw - function: lowercase strength: primary + locale: en_US + } + } + } + + + field syntaxcheck2b type string { + indexing: attribute + attribute { + sorting { + descending + function: raw strength: secondary + } + } + } + + field syntaxcheck3b type string { + indexing: attribute + attribute { + sorting { + ascending + function: lowercase strength: tertiary + } + } + } + + field syntaxcheck4b type string { + indexing: attribute + attribute { + sorting { + descending + function: uca strength: quaternary - strength: identical locale: en_US } } } + field syntaxcheck5b type string { + indexing: attribute + attribute { + sorting { + ascending + function: raw + strength: identical + } + } + } + field infieldonly type string { indexing: attribute sorting { diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index 567944d3b78..30dd3fc1d35 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -1,146 +1,158 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 105061838 -datatype[].structtype[].name "ns1" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "nf1" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "nf1s" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "nf2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 3474528 -datatype[].structtype[].name "s1" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "f1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f1s" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f2" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f3" -datatype[].structtype[].field[].datatype 5 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1497802371 -datatype[].maptype[].keytype 4 -datatype[].maptype[].valtype 2 -datatype[].id -1425630723 -datatype[].arraytype[].datatype 3474528 -datatype[].id 731395686 -datatype[].structtype[].name "streamingstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "coupleof" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "normalfields" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "a" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "m" -datatype[].structtype[].field[].datatype -1497802371 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "b" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c2" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c3" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "n" -datatype[].structtype[].field[].datatype 105061838 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "array1" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "array2" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "array3" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "subject" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "d" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "e" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "g" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "anothersummaryfield" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "snippet" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "snippet2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1433175737 -datatype[].documenttype[].name "streamingstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "a" -datatype[].documenttype[].fieldsets{[document]}.fields[] "array1" -datatype[].documenttype[].fieldsets{[document]}.fields[] "array2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "array3" -datatype[].documenttype[].fieldsets{[document]}.fields[] "b" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c3" -datatype[].documenttype[].fieldsets{[document]}.fields[] "coupleof" -datatype[].documenttype[].fieldsets{[document]}.fields[] "d" -datatype[].documenttype[].fieldsets{[document]}.fields[] "e" -datatype[].documenttype[].fieldsets{[document]}.fields[] "f" -datatype[].documenttype[].fieldsets{[document]}.fields[] "g" -datatype[].documenttype[].fieldsets{[document]}.fields[] "m" -datatype[].documenttype[].fieldsets{[document]}.fields[] "n" -datatype[].documenttype[].fieldsets{[document]}.fields[] "normalfields" -datatype[].documenttype[].fieldsets{[document]}.fields[] "subject" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "streamingstruct" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "a" +doctype[].fieldsets{[document]}.fields[] "array1" +doctype[].fieldsets{[document]}.fields[] "array2" +doctype[].fieldsets{[document]}.fields[] "array3" +doctype[].fieldsets{[document]}.fields[] "b" +doctype[].fieldsets{[document]}.fields[] "c" +doctype[].fieldsets{[document]}.fields[] "c2" +doctype[].fieldsets{[document]}.fields[] "c3" +doctype[].fieldsets{[document]}.fields[] "coupleof" +doctype[].fieldsets{[document]}.fields[] "d" +doctype[].fieldsets{[document]}.fields[] "e" +doctype[].fieldsets{[document]}.fields[] "f" +doctype[].fieldsets{[document]}.fields[] "g" +doctype[].fieldsets{[document]}.fields[] "m" +doctype[].fieldsets{[document]}.fields[] "n" +doctype[].fieldsets{[document]}.fields[] "normalfields" +doctype[].fieldsets{[document]}.fields[] "subject" +doctype[].arraytype[].idx 10020 +doctype[].arraytype[].elementtype 10017 +doctype[].arraytype[].idx 10021 +doctype[].arraytype[].elementtype 10017 +doctype[].arraytype[].idx 10022 +doctype[].arraytype[].elementtype 10017 +doctype[].maptype[].idx 10018 +doctype[].maptype[].keytype 10008 +doctype[].maptype[].valuetype 10012 +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "s1" +doctype[].structtype[].field[].name "f1" +doctype[].structtype[].field[].internalid 1911889118 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "f1s" +doctype[].structtype[].field[].internalid 624198897 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "f2" +doctype[].structtype[].field[].internalid 708327411 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "f3" +doctype[].structtype[].field[].internalid 2137295507 +doctype[].structtype[].field[].type 10004 +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "ns1" +doctype[].structtype[].field[].name "nf1" +doctype[].structtype[].field[].internalid 970725348 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "nf1s" +doctype[].structtype[].field[].internalid 47697102 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "nf2" +doctype[].structtype[].field[].internalid 1084090162 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "streamingstruct.header" +doctype[].structtype[].field[].name "coupleof" +doctype[].structtype[].field[].internalid 1611860736 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "normalfields" +doctype[].structtype[].field[].internalid 1898724546 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "a" +doctype[].structtype[].field[].internalid 1097803284 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "m" +doctype[].structtype[].field[].internalid 914298689 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "b" +doctype[].structtype[].field[].internalid 754773144 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "c" +doctype[].structtype[].field[].internalid 480045975 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "c2" +doctype[].structtype[].field[].internalid 504337916 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "c3" +doctype[].structtype[].field[].internalid 996353555 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "n" +doctype[].structtype[].field[].internalid 679223015 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "array1" +doctype[].structtype[].field[].internalid 1863296233 +doctype[].structtype[].field[].type 10020 +doctype[].structtype[].field[].name "array2" +doctype[].structtype[].field[].internalid 1491532076 +doctype[].structtype[].field[].type 10021 +doctype[].structtype[].field[].name "array3" +doctype[].structtype[].field[].internalid 1585514103 +doctype[].structtype[].field[].type 10022 +doctype[].structtype[].field[].name "subject" +doctype[].structtype[].field[].internalid 1677912730 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "d" +doctype[].structtype[].field[].internalid 679827909 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "e" +doctype[].structtype[].field[].internalid 416661641 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "f" +doctype[].structtype[].field[].internalid 1258112309 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "g" +doctype[].structtype[].field[].internalid 1091070635 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "snippet" +doctype[].structtype[].field[].internalid 1102597618 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "snippet2" +doctype[].structtype[].field[].internalid 1812076817 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "anothersummaryfield" +doctype[].structtype[].field[].internalid 1811005492 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg index cb56da169fa..e69de29bb2d 100644 --- a/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/onlydoc/documentmanager.cfg @@ -1,127 +0,0 @@ -enablecompression false -usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 105061838 -datatype[].structtype[].name "ns1" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "nf1" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "nf1s" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "nf2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 3474528 -datatype[].structtype[].name "s1" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "f1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f1s" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f2" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f3" -datatype[].structtype[].field[].datatype 5 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1497802371 -datatype[].maptype[].keytype 4 -datatype[].maptype[].valtype 2 -datatype[].id -1425630723 -datatype[].arraytype[].datatype 3474528 -datatype[].id 731395686 -datatype[].structtype[].name "streamingstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "coupleof" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "normalfields" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "a" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "m" -datatype[].structtype[].field[].datatype -1497802371 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "b" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c2" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c3" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "n" -datatype[].structtype[].field[].datatype 105061838 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "array1" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "array2" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "array3" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "subject" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "d" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "e" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "f" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "g" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 1858438651 -datatype[].structtype[].name "streamingstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].id 1433175737 -datatype[].documenttype[].name "streamingstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 1858438651 diff --git a/config-model/src/test/derived/structandfieldset/documentmanager.cfg b/config-model/src/test/derived/structandfieldset/documentmanager.cfg index 64677184fb7..85df5249e3a 100644 --- a/config-model/src/test/derived/structandfieldset/documentmanager.cfg +++ b/config-model/src/test/derived/structandfieldset/documentmanager.cfg @@ -1,54 +1,67 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -283419354 -datatype[].arraytype[].datatype 443162583 -datatype[].id 443162583 -datatype[].structtype[].name "person" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "first_name" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "last_name" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 306916075 -datatype[].structtype[].name "test.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "tag" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "people" -datatype[].structtype[].field[].datatype -283419354 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -877171244 -datatype[].documenttype[].name "test" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 306916075 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{default}.fields[] "people.first_name" -datatype[].documenttype[].fieldsets{default}.fields[] "tag" -datatype[].documenttype[].fieldsets{[]}.fields[] "people" -datatype[].documenttype[].fieldsets{[]}.fields[] "tag" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "test" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{default}.fields[] "people.first_name" +doctype[].fieldsets{default}.fields[] "tag" +doctype[].fieldsets{[document]}.fields[] "people" +doctype[].fieldsets{[document]}.fields[] "tag" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10018 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "person" +doctype[].structtype[].field[].name "first_name" +doctype[].structtype[].field[].internalid 1282582812 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "last_name" +doctype[].structtype[].field[].internalid 127888549 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "test.header" +doctype[].structtype[].field[].name "tag" +doctype[].structtype[].field[].internalid 431488563 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "people" +doctype[].structtype[].field[].internalid 1364826217 +doctype[].structtype[].field[].type 10017 diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index be5e473e460..b3b6a2587c1 100644 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -1,84 +1,88 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 109267174 -datatype[].structtype[].name "sct" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "s1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "s2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "s3" -datatype[].structtype[].field[].datatype 109267174 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "s4" -datatype[].structtype[].field[].datatype 97614088 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 97614088 -datatype[].structtype[].name "foo" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "s1" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 517946310 -datatype[].structtype[].name "annotation.banana" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "b" -datatype[].structtype[].field[].datatype 109267174 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "c" -datatype[].structtype[].field[].datatype 97614088 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1244829667 -datatype[].arraytype[].datatype 109267174 -datatype[].id -364910881 -datatype[].structtype[].name "annotationsimplicitstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "structfield" -datatype[].structtype[].field[].datatype 109267174 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "structarrayfield" -datatype[].structtype[].field[].datatype -1244829667 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -2099544992 -datatype[].documenttype[].name "annotationsimplicitstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct -364910881 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "structarrayfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "structfield" -annotationtype[].id -269517759 -annotationtype[].name "banana" -annotationtype[].datatype 517946310 +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "annotationsimplicitstruct" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "structarrayfield" +doctype[].fieldsets{[document]}.fields[] "structfield" +doctype[].arraytype[].idx 10019 +doctype[].arraytype[].elementtype 10017 +doctype[].annotationtype[].idx 10021 +doctype[].annotationtype[].name "banana" +doctype[].annotationtype[].internalid -269517759 +doctype[].annotationtype[].datatype 10020 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "foo" +doctype[].structtype[].field[].name "s1" +doctype[].structtype[].field[].internalid 475255687 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "sct" +doctype[].structtype[].field[].name "s1" +doctype[].structtype[].field[].internalid 2146820765 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "s2" +doctype[].structtype[].field[].internalid 45366795 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "s3" +doctype[].structtype[].field[].internalid 939491122 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "s4" +doctype[].structtype[].field[].internalid 1572342940 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "annotationsimplicitstruct.header" +doctype[].structtype[].field[].name "structfield" +doctype[].structtype[].field[].internalid 486207386 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "structarrayfield" +doctype[].structtype[].field[].internalid 335048518 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "annotation.banana" +doctype[].structtype[].field[].name "b" +doctype[].structtype[].field[].internalid 1964392128 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "c" +doctype[].structtype[].field[].internalid 250751515 +doctype[].structtype[].field[].type 10018 diff --git a/config-model/src/test/derived/structinheritance/documentmanager.cfg b/config-model/src/test/derived/structinheritance/documentmanager.cfg index 20994bacca2..ca11481afa4 100644 --- a/config-model/src/test/derived/structinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/structinheritance/documentmanager.cfg @@ -1,71 +1,72 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id -1396204461 -datatype[1].structtype[0].name "base" -datatype[1].structtype[0].version 0 -datatype[1].structtype[0].compresstype NONE -datatype[1].structtype[0].compresslevel 0 -datatype[1].structtype[0].compressthreshold 95 -datatype[1].structtype[0].compressminsize 800 -datatype[1].structtype[0].field[0].name "name" -datatype[1].structtype[0].field[0].datatype 2 -datatype[1].structtype[0].field[0].detailedtype "" -datatype[2].id 746267614 -datatype[2].structtype[0].name "child" -datatype[2].structtype[0].version 0 -datatype[2].structtype[0].compresstype NONE -datatype[2].structtype[0].compresslevel 0 -datatype[2].structtype[0].compressthreshold 95 -datatype[2].structtype[0].compressminsize 800 -datatype[2].structtype[0].field[0].name "age" -datatype[2].structtype[0].field[0].datatype 0 -datatype[2].structtype[0].field[0].detailedtype "" -datatype[2].structtype[0].inherits[0].name "base" -datatype[2].structtype[0].inherits[0].version 0 -datatype[3].id 1811766610 -datatype[3].structtype[0].name "grandchild" -datatype[3].structtype[0].version 0 -datatype[3].structtype[0].compresstype NONE -datatype[3].structtype[0].compresslevel 0 -datatype[3].structtype[0].compressthreshold 95 -datatype[3].structtype[0].compressminsize 800 -datatype[3].structtype[0].field[0].name "toy" -datatype[3].structtype[0].field[0].datatype 2 -datatype[3].structtype[0].field[0].detailedtype "" -datatype[3].structtype[0].inherits[0].name "child" -datatype[3].structtype[0].inherits[0].version 0 -datatype[4].id -2142109237 -datatype[4].structtype[0].name "simple.header" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[4].structtype[0].field[0].name "f1" -datatype[4].structtype[0].field[0].datatype 746267614 -datatype[4].structtype[0].field[0].detailedtype "" -datatype[4].structtype[0].field[1].name "f2" -datatype[4].structtype[0].field[1].datatype 1811766610 -datatype[4].structtype[0].field[1].detailedtype "" -datatype[5].id 485659380 -datatype[5].documenttype[0].name "simple" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct -2142109237 -datatype[5].documenttype[0].bodystruct 0 -datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "f1" -datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "f2" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "simple" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "f1" +doctype[].fieldsets{[document]}.fields[] "f2" +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "base" +doctype[].structtype[].field[].name "name" +doctype[].structtype[].field[].internalid 1160796772 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "child" +doctype[].structtype[].inherits[].type 10018 +doctype[].structtype[].field[].name "age" +doctype[].structtype[].field[].internalid 1862473705 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "grandchild" +doctype[].structtype[].inherits[].type 10017 +doctype[].structtype[].field[].name "toy" +doctype[].structtype[].field[].internalid 536645790 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "simple.header" +doctype[].structtype[].field[].name "f1" +doctype[].structtype[].field[].internalid 750623154 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "f2" +doctype[].structtype[].field[].internalid 1523850983 +doctype[].structtype[].field[].type 10019 diff --git a/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg b/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg index f121eb7628a..e69de29bb2d 100644 --- a/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg +++ b/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg @@ -1,145 +0,0 @@ -enablecompression false -usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].id 105061838 -datatype[].structtype[].name "ns1" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "nf1" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "nf1s" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "nf2" -datatype[].structtype[].field[].datatype 2 -datatype[].id 3474528 -datatype[].structtype[].name "s1" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "f1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "f1s" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "f2" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].name "f3" -datatype[].structtype[].field[].datatype 5 -datatype[].id -1497802371 -datatype[].maptype[].keytype 4 -datatype[].maptype[].valtype 2 -datatype[].id -1425630723 -datatype[].arraytype[].datatype 3474528 -datatype[].id 731395686 -datatype[].structtype[].name "streamingstruct.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "coupleof" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "normalfields" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "a" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "m" -datatype[].structtype[].field[].datatype -1497802371 -datatype[].structtype[].field[].name "b" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "c" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "c2" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "c3" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "n" -datatype[].structtype[].field[].datatype 105061838 -datatype[].structtype[].field[].name "array1" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].name "array2" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].name "array3" -datatype[].structtype[].field[].datatype -1425630723 -datatype[].structtype[].field[].name "subject" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "d" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "e" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "f" -datatype[].structtype[].field[].datatype 3474528 -datatype[].structtype[].field[].name "g" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "anothersummaryfield" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "snippet" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "snippet2" -datatype[].structtype[].field[].datatype 2 -datatype[].id 1433175737 -datatype[].documenttype[].name "streamingstruct" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "a" -datatype[].documenttype[].fieldsets{[document]}.fields[] "array1" -datatype[].documenttype[].fieldsets{[document]}.fields[] "array2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "array3" -datatype[].documenttype[].fieldsets{[document]}.fields[] "b" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "c3" -datatype[].documenttype[].fieldsets{[document]}.fields[] "coupleof" -datatype[].documenttype[].fieldsets{[document]}.fields[] "d" -datatype[].documenttype[].fieldsets{[document]}.fields[] "e" -datatype[].documenttype[].fieldsets{[document]}.fields[] "f" -datatype[].documenttype[].fieldsets{[document]}.fields[] "g" -datatype[].documenttype[].fieldsets{[document]}.fields[] "m" -datatype[].documenttype[].fieldsets{[document]}.fields[] "n" -datatype[].documenttype[].fieldsets{[document]}.fields[] "normalfields" -datatype[].documenttype[].fieldsets{[document]}.fields[] "subject" -datatype[].id -995681764 -datatype[].structtype[].name "pair" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "key" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].name "value" -datatype[].structtype[].field[].datatype 2 -datatype[].id 355471259 -datatype[].structtype[].name "whatever.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "f1" -datatype[].structtype[].field[].datatype -995681764 -datatype[].id -778211548 -datatype[].documenttype[].name "whatever" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 355471259 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "f1" diff --git a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd index 32823e46592..0636e7a537e 100644 --- a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd +++ b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd @@ -21,6 +21,7 @@ search streamingstruct { # Allow default matchtypes in struct. Can be overridden. # No index/attribute related stuff. It is only a datatype definition. } + struct ns1 { field nf1 type s1 { } field nf1s type s1 { match: substring } diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index ffdf090f761..f74ce87e6cb 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -1,255 +1,286 @@ enablecompression false usev8geopositions false -datatype[].id 1381038251 -datatype[].structtype[].name "position" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "x" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "y" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1865479609 -datatype[].maptype[].keytype 2 -datatype[].maptype[].valtype 4 -datatype[].id 294108848 -datatype[].structtype[].name "folder" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "Version" -datatype[].structtype[].field[].datatype 0 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "Name" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "FlagsCounter" -datatype[].structtype[].field[].datatype -1865479609 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "anotherfolder" -datatype[].structtype[].field[].datatype 294108848 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 109267174 -datatype[].structtype[].name "sct" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "s1" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "s2" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id 49942803 -datatype[].arraytype[].datatype 16 -datatype[].id 339965458 -datatype[].maptype[].keytype 2 -datatype[].maptype[].valtype 2 -datatype[].id -2092985853 -datatype[].structtype[].name "mystruct" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "bytearr" -datatype[].structtype[].field[].datatype 49942803 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "mymap" -datatype[].structtype[].field[].datatype 339965458 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "title" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "structfield" -datatype[].structtype[].field[].datatype 2 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -1245117006 -datatype[].arraytype[].datatype 0 -datatype[].id 1328286588 -datatype[].weightedsettype[].datatype 2 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero false -datatype[].id 2125328771 -datatype[].weightedsettype[].datatype 2 -datatype[].weightedsettype[].createifnonexistant false -datatype[].weightedsettype[].removeifzero true -datatype[].id 2065577986 -datatype[].weightedsettype[].datatype 2 -datatype[].weightedsettype[].createifnonexistant true -datatype[].weightedsettype[].removeifzero false -datatype[].id -1244829667 -datatype[].arraytype[].datatype 109267174 -datatype[].id -1584287606 -datatype[].maptype[].keytype 2 -datatype[].maptype[].valtype 0 -datatype[].id 2125154557 -datatype[].maptype[].keytype 2 -datatype[].maptype[].valtype 1 -datatype[].id -1715531035 -datatype[].maptype[].keytype 0 -datatype[].maptype[].valtype 4 -datatype[].id 2138385264 -datatype[].maptype[].keytype 0 -datatype[].maptype[].valtype 5 -datatype[].id 435886609 -datatype[].maptype[].keytype 2 -datatype[].maptype[].valtype -1245117006 -datatype[].id -1486737430 -datatype[].arraytype[].datatype 2 -datatype[].id 1707615575 -datatype[].arraytype[].datatype -1486737430 -datatype[].id -794985308 -datatype[].arraytype[].datatype 1707615575 -datatype[].id 69621385 -datatype[].arraytype[].datatype 339965458 -datatype[].id 1901258752 -datatype[].maptype[].keytype 0 -datatype[].maptype[].valtype -2092985853 -datatype[].id 759956026 -datatype[].arraytype[].datatype -2092985853 -datatype[].id -389833101 -datatype[].maptype[].keytype 0 -datatype[].maptype[].valtype 294108848 -datatype[].id 1328581348 -datatype[].structtype[].name "types.header" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "abyte" -datatype[].structtype[].field[].datatype 16 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "along" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "abool" -datatype[].structtype[].field[].datatype 6 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "ashortfloat" -datatype[].structtype[].field[].datatype 7 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "arrayfield" -datatype[].structtype[].field[].datatype -1245117006 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "setfield" -datatype[].structtype[].field[].datatype 1328286588 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "setfield2" -datatype[].structtype[].field[].datatype 18 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "setfield3" -datatype[].structtype[].field[].datatype 2125328771 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "setfield4" -datatype[].structtype[].field[].datatype 2065577986 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "tagfield" -datatype[].structtype[].field[].datatype 18 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "structfield" -datatype[].structtype[].field[].datatype 109267174 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "structarrayfield" -datatype[].structtype[].field[].datatype -1244829667 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "stringmapfield" -datatype[].structtype[].field[].datatype 339965458 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "intmapfield" -datatype[].structtype[].field[].datatype -1584287606 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "floatmapfield" -datatype[].structtype[].field[].datatype 2125154557 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "longmapfield" -datatype[].structtype[].field[].datatype -1715531035 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "doublemapfield" -datatype[].structtype[].field[].datatype 2138385264 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "arraymapfield" -datatype[].structtype[].field[].datatype 435886609 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "arrarr" -datatype[].structtype[].field[].datatype -794985308 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "maparr" -datatype[].structtype[].field[].datatype 69621385 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "mystructfield" -datatype[].structtype[].field[].datatype -2092985853 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "mystructmap" -datatype[].structtype[].field[].datatype 1901258752 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "mystructarr" -datatype[].structtype[].field[].datatype 759956026 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "Folders" -datatype[].structtype[].field[].datatype -389833101 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "juletre" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "album0" -datatype[].structtype[].field[].datatype 18 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "album1" -datatype[].structtype[].field[].datatype 18 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "other" -datatype[].structtype[].field[].datatype 4 -datatype[].structtype[].field[].detailedtype "" -datatype[].structtype[].field[].name "complexarray" -datatype[].structtype[].field[].datatype 1416345047 -datatype[].structtype[].field[].detailedtype "" -datatype[].id -372512406 -datatype[].maptype[].keytype 0 -datatype[].maptype[].valtype 1707615575 -datatype[].id 1416345047 -datatype[].arraytype[].datatype -372512406 -datatype[].id -853072901 -datatype[].documenttype[].name "types" -datatype[].documenttype[].version 0 -datatype[].documenttype[].inherits[].name "document" -datatype[].documenttype[].inherits[].version 0 -datatype[].documenttype[].headerstruct 1328581348 -datatype[].documenttype[].bodystruct 0 -datatype[].documenttype[].fieldsets{[document]}.fields[] "Folders" -datatype[].documenttype[].fieldsets{[document]}.fields[] "abool" -datatype[].documenttype[].fieldsets{[document]}.fields[] "abyte" -datatype[].documenttype[].fieldsets{[document]}.fields[] "album0" -datatype[].documenttype[].fieldsets{[document]}.fields[] "album1" -datatype[].documenttype[].fieldsets{[document]}.fields[] "along" -datatype[].documenttype[].fieldsets{[document]}.fields[] "arrarr" -datatype[].documenttype[].fieldsets{[document]}.fields[] "arrayfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "arraymapfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "ashortfloat" -datatype[].documenttype[].fieldsets{[document]}.fields[] "complexarray" -datatype[].documenttype[].fieldsets{[document]}.fields[] "doublemapfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "floatmapfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "intmapfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "juletre" -datatype[].documenttype[].fieldsets{[document]}.fields[] "longmapfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "maparr" -datatype[].documenttype[].fieldsets{[document]}.fields[] "mystructarr" -datatype[].documenttype[].fieldsets{[document]}.fields[] "mystructfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "mystructmap" -datatype[].documenttype[].fieldsets{[document]}.fields[] "setfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "setfield2" -datatype[].documenttype[].fieldsets{[document]}.fields[] "setfield3" -datatype[].documenttype[].fieldsets{[document]}.fields[] "setfield4" -datatype[].documenttype[].fieldsets{[document]}.fields[] "stringmapfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "structarrayfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "structfield" -datatype[].documenttype[].fieldsets{[document]}.fields[] "tagfield" +doctype[].name "document" +doctype[].idx 10000 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].name "types" +doctype[].idx 10015 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "Folders" +doctype[].fieldsets{[document]}.fields[] "abool" +doctype[].fieldsets{[document]}.fields[] "abyte" +doctype[].fieldsets{[document]}.fields[] "album0" +doctype[].fieldsets{[document]}.fields[] "album1" +doctype[].fieldsets{[document]}.fields[] "along" +doctype[].fieldsets{[document]}.fields[] "arrarr" +doctype[].fieldsets{[document]}.fields[] "arrayfield" +doctype[].fieldsets{[document]}.fields[] "arraymapfield" +doctype[].fieldsets{[document]}.fields[] "ashortfloat" +doctype[].fieldsets{[document]}.fields[] "complexarray" +doctype[].fieldsets{[document]}.fields[] "doublemapfield" +doctype[].fieldsets{[document]}.fields[] "floatmapfield" +doctype[].fieldsets{[document]}.fields[] "intmapfield" +doctype[].fieldsets{[document]}.fields[] "juletre" +doctype[].fieldsets{[document]}.fields[] "longmapfield" +doctype[].fieldsets{[document]}.fields[] "maparr" +doctype[].fieldsets{[document]}.fields[] "mystructarr" +doctype[].fieldsets{[document]}.fields[] "mystructfield" +doctype[].fieldsets{[document]}.fields[] "mystructmap" +doctype[].fieldsets{[document]}.fields[] "setfield" +doctype[].fieldsets{[document]}.fields[] "setfield2" +doctype[].fieldsets{[document]}.fields[] "setfield3" +doctype[].fieldsets{[document]}.fields[] "setfield4" +doctype[].fieldsets{[document]}.fields[] "stringmapfield" +doctype[].fieldsets{[document]}.fields[] "structarrayfield" +doctype[].fieldsets{[document]}.fields[] "structfield" +doctype[].fieldsets{[document]}.fields[] "tagfield" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10007 +doctype[].arraytype[].idx 10024 +doctype[].arraytype[].elementtype 10023 +doctype[].arraytype[].idx 10031 +doctype[].arraytype[].elementtype 10007 +doctype[].arraytype[].idx 10032 +doctype[].arraytype[].elementtype 10033 +doctype[].arraytype[].idx 10033 +doctype[].arraytype[].elementtype 10034 +doctype[].arraytype[].idx 10034 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].idx 10035 +doctype[].arraytype[].elementtype 10036 +doctype[].arraytype[].idx 10037 +doctype[].arraytype[].elementtype 10038 +doctype[].arraytype[].idx 10039 +doctype[].arraytype[].elementtype 10040 +doctype[].arraytype[].idx 10040 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].idx 10042 +doctype[].arraytype[].elementtype 10003 +doctype[].arraytype[].idx 10045 +doctype[].arraytype[].elementtype 10041 +doctype[].maptype[].idx 10025 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10012 +doctype[].maptype[].idx 10026 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10007 +doctype[].maptype[].idx 10027 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10005 +doctype[].maptype[].idx 10028 +doctype[].maptype[].keytype 10007 +doctype[].maptype[].valuetype 10008 +doctype[].maptype[].idx 10029 +doctype[].maptype[].keytype 10007 +doctype[].maptype[].valuetype 10004 +doctype[].maptype[].idx 10030 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10031 +doctype[].maptype[].idx 10036 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10012 +doctype[].maptype[].idx 10038 +doctype[].maptype[].keytype 10007 +doctype[].maptype[].valuetype 10039 +doctype[].maptype[].idx 10043 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10012 +doctype[].maptype[].idx 10044 +doctype[].maptype[].keytype 10007 +doctype[].maptype[].valuetype 10041 +doctype[].maptype[].idx 10046 +doctype[].maptype[].keytype 10007 +doctype[].maptype[].valuetype 10047 +doctype[].maptype[].idx 10048 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10008 +doctype[].wsettype[].idx 10018 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10019 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].idx 10020 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent false +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].idx 10021 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero false +doctype[].wsettype[].idx 10022 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].idx 10049 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].idx 10050 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].structtype[].idx 10023 +doctype[].structtype[].name "sct" +doctype[].structtype[].field[].name "s1" +doctype[].structtype[].field[].internalid 2146820765 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "s2" +doctype[].structtype[].field[].internalid 45366795 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10041 +doctype[].structtype[].name "mystruct" +doctype[].structtype[].field[].name "bytearr" +doctype[].structtype[].field[].internalid 1079701754 +doctype[].structtype[].field[].type 10042 +doctype[].structtype[].field[].name "mymap" +doctype[].structtype[].field[].internalid 1954178122 +doctype[].structtype[].field[].type 10043 +doctype[].structtype[].field[].name "title" +doctype[].structtype[].field[].internalid 567626448 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "structfield" +doctype[].structtype[].field[].internalid 1726890940 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].idx 10047 +doctype[].structtype[].name "folder" +doctype[].structtype[].field[].name "Version" +doctype[].structtype[].field[].internalid 64430502 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "Name" +doctype[].structtype[].field[].internalid 2002760220 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "FlagsCounter" +doctype[].structtype[].field[].internalid 1741227606 +doctype[].structtype[].field[].type 10048 +doctype[].structtype[].field[].name "anotherfolder" +doctype[].structtype[].field[].internalid 1582421848 +doctype[].structtype[].field[].type 10047 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "types.header" +doctype[].structtype[].field[].name "abyte" +doctype[].structtype[].field[].internalid 110138156 +doctype[].structtype[].field[].type 10003 +doctype[].structtype[].field[].name "along" +doctype[].structtype[].field[].internalid 1206464520 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "abool" +doctype[].structtype[].field[].internalid 492328000 +doctype[].structtype[].field[].type 10002 +doctype[].structtype[].field[].name "ashortfloat" +doctype[].structtype[].field[].internalid 1012106297 +doctype[].structtype[].field[].type 10006 +doctype[].structtype[].field[].name "arrayfield" +doctype[].structtype[].field[].internalid 965790107 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "setfield" +doctype[].structtype[].field[].internalid 761581914 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "setfield2" +doctype[].structtype[].field[].internalid 1066659198 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "setfield3" +doctype[].structtype[].field[].internalid 1180155772 +doctype[].structtype[].field[].type 10020 +doctype[].structtype[].field[].name "setfield4" +doctype[].structtype[].field[].internalid 1254131631 +doctype[].structtype[].field[].type 10021 +doctype[].structtype[].field[].name "tagfield" +doctype[].structtype[].field[].internalid 1653562069 +doctype[].structtype[].field[].type 10022 +doctype[].structtype[].field[].name "structfield" +doctype[].structtype[].field[].internalid 486207386 +doctype[].structtype[].field[].type 10023 +doctype[].structtype[].field[].name "structarrayfield" +doctype[].structtype[].field[].internalid 335048518 +doctype[].structtype[].field[].type 10024 +doctype[].structtype[].field[].name "stringmapfield" +doctype[].structtype[].field[].internalid 117465687 +doctype[].structtype[].field[].type 10025 +doctype[].structtype[].field[].name "intmapfield" +doctype[].structtype[].field[].internalid 121004462 +doctype[].structtype[].field[].type 10026 +doctype[].structtype[].field[].name "floatmapfield" +doctype[].structtype[].field[].internalid 1239120925 +doctype[].structtype[].field[].type 10027 +doctype[].structtype[].field[].name "longmapfield" +doctype[].structtype[].field[].internalid 477718745 +doctype[].structtype[].field[].type 10028 +doctype[].structtype[].field[].name "doublemapfield" +doctype[].structtype[].field[].internalid 877047192 +doctype[].structtype[].field[].type 10029 +doctype[].structtype[].field[].name "arraymapfield" +doctype[].structtype[].field[].internalid 1670805928 +doctype[].structtype[].field[].type 10030 +doctype[].structtype[].field[].name "arrarr" +doctype[].structtype[].field[].internalid 1962567166 +doctype[].structtype[].field[].type 10032 +doctype[].structtype[].field[].name "maparr" +doctype[].structtype[].field[].internalid 904375219 +doctype[].structtype[].field[].type 10035 +doctype[].structtype[].field[].name "complexarray" +doctype[].structtype[].field[].internalid 795629533 +doctype[].structtype[].field[].type 10037 +doctype[].structtype[].field[].name "mystructfield" +doctype[].structtype[].field[].internalid 1348513378 +doctype[].structtype[].field[].type 10041 +doctype[].structtype[].field[].name "mystructmap" +doctype[].structtype[].field[].internalid 1511423250 +doctype[].structtype[].field[].type 10044 +doctype[].structtype[].field[].name "mystructarr" +doctype[].structtype[].field[].internalid 595856991 +doctype[].structtype[].field[].type 10045 +doctype[].structtype[].field[].name "Folders" +doctype[].structtype[].field[].internalid 34575524 +doctype[].structtype[].field[].type 10046 +doctype[].structtype[].field[].name "juletre" +doctype[].structtype[].field[].internalid 1039981530 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "album0" +doctype[].structtype[].field[].internalid 764312262 +doctype[].structtype[].field[].type 10049 +doctype[].structtype[].field[].name "album1" +doctype[].structtype[].field[].internalid 1967160809 +doctype[].structtype[].field[].type 10050 +doctype[].structtype[].field[].name "other" +doctype[].structtype[].field[].internalid 2443357 +doctype[].structtype[].field[].type 10008 diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index 2db4ead180b..7f43cd67a6b 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -96,7 +96,7 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "tagfield" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "tagfield" -indexinfo[].command[].command "type tag" +indexinfo[].command[].command "type WeightedSet<string>" indexinfo[].command[].indexname "structfield.s1" indexinfo[].command[].command "index" indexinfo[].command[].indexname "structfield.s1" @@ -704,4 +704,4 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "pst_sta_boldingoff_nomatch_tag_01" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "pst_sta_boldingoff_nomatch_tag_01" -indexinfo[].command[].command "type tag" +indexinfo[].command[].command "type WeightedSet<string>" diff --git a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg index a7a4c675311..ea56800ea1c 100644 --- a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg +++ b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg @@ -4,41 +4,41 @@ doctype[0].name "document" doctype[0].idx 10000 doctype[0].contentstruct 10001 doctype[0].primitivetype[0].idx 10002 -doctype[0].primitivetype[0].name "byte" +doctype[0].primitivetype[0].name "bool" doctype[0].primitivetype[1].idx 10003 -doctype[0].primitivetype[1].name "int" +doctype[0].primitivetype[1].name "byte" doctype[0].primitivetype[2].idx 10004 -doctype[0].primitivetype[2].name "long" +doctype[0].primitivetype[2].name "double" doctype[0].primitivetype[3].idx 10005 -doctype[0].primitivetype[3].name "string" +doctype[0].primitivetype[3].name "float" doctype[0].primitivetype[4].idx 10006 -doctype[0].primitivetype[4].name "raw" -doctype[0].primitivetype[5].idx 10008 -doctype[0].primitivetype[5].name "float" -doctype[0].primitivetype[6].idx 10009 -doctype[0].primitivetype[6].name "double" -doctype[0].primitivetype[7].idx 10011 -doctype[0].primitivetype[7].name "uri" -doctype[0].primitivetype[8].idx 10012 -doctype[0].primitivetype[8].name "predicate" -doctype[0].primitivetype[9].idx 10013 -doctype[0].primitivetype[9].name "bool" +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" doctype[0].primitivetype[10].idx 10014 -doctype[0].primitivetype[10].name "float16" -doctype[0].wsettype[0].idx 10007 -doctype[0].wsettype[0].elementtype 10005 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 doctype[0].wsettype[0].createifnonexistent true doctype[0].wsettype[0].removeifzero true doctype[0].structtype[0].idx 10001 doctype[0].structtype[0].name "document.header" -doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].idx 10009 doctype[0].structtype[1].name "position" doctype[0].structtype[1].field[0].name "x" doctype[0].structtype[1].field[0].internalid 914677694 -doctype[0].structtype[1].field[0].type 10003 +doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 -doctype[0].structtype[1].field[1].type 10003 +doctype[0].structtype[1].field[1].type 10007 doctype[1].name "book" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 @@ -60,10 +60,10 @@ doctype[2].structtype[0].idx 10018 doctype[2].structtype[0].name "music.header" doctype[2].structtype[0].field[0].name "intfield" doctype[2].structtype[0].field[0].internalid 435380425 -doctype[2].structtype[0].field[0].type 10003 +doctype[2].structtype[0].field[0].type 10007 doctype[2].structtype[0].field[1].name "stringfield" doctype[2].structtype[0].field[1].internalid 1182460484 -doctype[2].structtype[0].field[1].type 10005 +doctype[2].structtype[0].field[1].type 10012 doctype[2].structtype[0].field[2].name "longfield" doctype[2].structtype[0].field[2].internalid 1589309697 -doctype[2].structtype[0].field[2].type 10004 +doctype[2].structtype[0].field[2].type 10008 diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index 82a30012a07..ea56800ea1c 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -1,59 +1,69 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id -1344444812 -datatype[1].structtype[0].name "book.header" -datatype[1].structtype[0].version 0 -datatype[1].structtype[0].compresstype NONE -datatype[1].structtype[0].compresslevel 0 -datatype[1].structtype[0].compressthreshold 95 -datatype[1].structtype[0].compressminsize 800 -datatype[1].structtype[0].field[0].name "soundtrack" -datatype[1].structtype[0].field[0].datatype 1412693671 -datatype[1].structtype[0].field[0].detailedtype "" -datatype[2].id 1412693671 -datatype[2].documenttype[0].name "music" -datatype[2].documenttype[0].version 0 -datatype[2].documenttype[0].inherits[0].name "document" -datatype[2].documenttype[0].inherits[0].version 0 -datatype[2].documenttype[0].headerstruct -1910204744 -datatype[2].documenttype[0].bodystruct 0 -datatype[2].documenttype[0].fieldsets{[document]}.fields[0] "intfield" -datatype[2].documenttype[0].fieldsets{[document]}.fields[1] "longfield" -datatype[2].documenttype[0].fieldsets{[document]}.fields[2] "stringfield" -datatype[3].id -1383388565 -datatype[3].documenttype[0].name "book" -datatype[3].documenttype[0].version 0 -datatype[3].documenttype[0].inherits[0].name "document" -datatype[3].documenttype[0].inherits[0].version 0 -datatype[3].documenttype[0].headerstruct -1344444812 -datatype[3].documenttype[0].bodystruct 0 -datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack" -datatype[4].id -1910204744 -datatype[4].structtype[0].name "music.header" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[4].structtype[0].field[0].name "intfield" -datatype[4].structtype[0].field[0].datatype 0 -datatype[4].structtype[0].field[0].detailedtype "" -datatype[4].structtype[0].field[1].name "stringfield" -datatype[4].structtype[0].field[1].datatype 2 -datatype[4].structtype[0].field[1].detailedtype "" -datatype[4].structtype[0].field[2].name "longfield" -datatype[4].structtype[0].field[2].datatype 4 -datatype[4].structtype[0].field[2].detailedtype "" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "book" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "soundtrack" +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "book.header" +doctype[1].structtype[0].field[0].name "soundtrack" +doctype[1].structtype[0].field[0].internalid 1258961213 +doctype[1].structtype[0].field[0].type 10017 +doctype[2].name "music" +doctype[2].idx 10017 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "intfield" +doctype[2].fieldsets{[document]}.fields[1] "longfield" +doctype[2].fieldsets{[document]}.fields[2] "stringfield" +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "music.header" +doctype[2].structtype[0].field[0].name "intfield" +doctype[2].structtype[0].field[0].internalid 435380425 +doctype[2].structtype[0].field[0].type 10007 +doctype[2].structtype[0].field[1].name "stringfield" +doctype[2].structtype[0].field[1].internalid 1182460484 +doctype[2].structtype[0].field[1].type 10012 +doctype[2].structtype[0].field[2].name "longfield" +doctype[2].structtype[0].field[2].internalid 1589309697 +doctype[2].structtype[0].field[2].type 10008 diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index b5b90245858..34383084625 100644 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -1,67 +1,74 @@ enablecompression false usev8geopositions false -datatype[0].id 1381038251 -datatype[0].structtype[0].name "position" -datatype[0].structtype[0].version 0 -datatype[0].structtype[0].compresstype NONE -datatype[0].structtype[0].compresslevel 0 -datatype[0].structtype[0].compressthreshold 95 -datatype[0].structtype[0].compressminsize 800 -datatype[0].structtype[0].field[0].name "x" -datatype[0].structtype[0].field[0].datatype 0 -datatype[0].structtype[0].field[0].detailedtype "" -datatype[0].structtype[0].field[1].name "y" -datatype[0].structtype[0].field[1].datatype 0 -datatype[0].structtype[0].field[1].detailedtype "" -datatype[1].id -1245205573 -datatype[1].arraytype[0].datatype 97614088 -datatype[2].id 93505813 -datatype[2].structtype[0].name "bar" -datatype[2].structtype[0].version 0 -datatype[2].structtype[0].compresstype NONE -datatype[2].structtype[0].compresslevel 0 -datatype[2].structtype[0].compressthreshold 95 -datatype[2].structtype[0].compressminsize 800 -datatype[2].structtype[0].field[0].name "humbe" -datatype[2].structtype[0].field[0].datatype 97614088 -datatype[2].structtype[0].field[0].detailedtype "" -datatype[3].id 97614088 -datatype[3].structtype[0].name "foo" -datatype[3].structtype[0].version 0 -datatype[3].structtype[0].compresstype NONE -datatype[3].structtype[0].compresslevel 0 -datatype[3].structtype[0].compressthreshold 95 -datatype[3].structtype[0].compressminsize 800 -datatype[3].structtype[0].field[0].name "fubar" -datatype[3].structtype[0].field[0].datatype 0 -datatype[3].structtype[0].field[0].detailedtype "" -datatype[3].structtype[0].field[1].name "bar" -datatype[3].structtype[0].field[1].id[0].id 1 -datatype[3].structtype[0].field[1].datatype 2 -datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id -1910204744 -datatype[4].structtype[0].name "music.header" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[4].structtype[0].field[0].name "mystruct" -datatype[4].structtype[0].field[0].datatype 97614088 -datatype[4].structtype[0].field[0].detailedtype "" -datatype[4].structtype[0].field[1].name "arraystruct" -datatype[4].structtype[0].field[1].datatype -1245205573 -datatype[4].structtype[0].field[1].detailedtype "" -datatype[4].structtype[0].field[2].name "advanced" -datatype[4].structtype[0].field[2].datatype 93505813 -datatype[4].structtype[0].field[2].detailedtype "" -datatype[5].id 1412693671 -datatype[5].documenttype[0].name "music" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct -1910204744 -datatype[5].documenttype[0].bodystruct 0 -datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "advanced" -datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "arraystruct" -datatype[5].documenttype[0].fieldsets{[document]}.fields[2] "mystruct" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "music" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "advanced" +doctype[1].fieldsets{[document]}.fields[1] "arraystruct" +doctype[1].fieldsets{[document]}.fields[2] "mystruct" +doctype[1].arraytype[0].idx 10018 +doctype[1].arraytype[0].elementtype 10017 +doctype[1].structtype[0].idx 10017 +doctype[1].structtype[0].name "foo" +doctype[1].structtype[0].field[0].name "fubar" +doctype[1].structtype[0].field[0].internalid 445123100 +doctype[1].structtype[0].field[0].type 10007 +doctype[1].structtype[0].field[1].name "bar" +doctype[1].structtype[0].field[1].internalid 1 +doctype[1].structtype[0].field[1].type 10012 +doctype[1].structtype[1].idx 10019 +doctype[1].structtype[1].name "bar" +doctype[1].structtype[1].field[0].name "humbe" +doctype[1].structtype[1].field[0].internalid 666115807 +doctype[1].structtype[1].field[0].type 10017 +doctype[1].structtype[2].idx 10016 +doctype[1].structtype[2].name "music.header" +doctype[1].structtype[2].field[0].name "mystruct" +doctype[1].structtype[2].field[0].internalid 595153726 +doctype[1].structtype[2].field[0].type 10017 +doctype[1].structtype[2].field[1].name "arraystruct" +doctype[1].structtype[2].field[1].internalid 932640198 +doctype[1].structtype[2].field[1].type 10018 +doctype[1].structtype[2].field[2].name "advanced" +doctype[1].structtype[2].field[2].internalid 659465951 +doctype[1].structtype[2].field[2].type 10019 diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java index 81a44261daf..8edbd789a27 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java @@ -156,8 +156,8 @@ public class DocumentGraphValidatorTest { @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); fromDocument.addField(refField); Map<String, DocumentReference> originalMap = fromDocument.getDocumentReferences().get().referenceMap(); HashMap<String, DocumentReference> modifiedMap = new HashMap<>(originalMap); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 66f1850bd10..4fa145afae9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -38,12 +38,12 @@ public class DocumentReferenceResolverTest { barSchema.addDocument(barDocument); // Create foo document with document reference to bar and add another field - SDField fooRefToBarField = new SDField - ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToBarField); - SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToBarField); + SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); fooSchema.addDocument(fooDocument); @@ -62,11 +62,12 @@ public class DocumentReferenceResolverTest { @Test public void throws_user_friendly_exception_if_referenced_document_does_not_exist() { // Create foo document with document reference to non-existing document bar + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); SDField fooRefToBarField = new SDField( + fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); AttributeUtils.addAttributeAspect(fooRefToBarField); - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); @@ -86,10 +87,10 @@ public class DocumentReferenceResolverTest { barSchema.addDocument(barDocument); // Create foo document with document reference to bar - SDField fooRefToBarField = new SDField - ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java index 7d2386030da..7e708f93a96 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -22,16 +22,18 @@ public class ImportedFieldsEnumeratorTest { String PARENT = "parent"; Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); - var parentField = new SDField("their_field", DataType.INT); + var parentField = new SDField(parentDocument, "their_field", DataType.INT); AttributeUtils.addAttributeAspect(parentField); parentDocument.addField(parentField); parentSchema.addDocument(parentDocument); String FOO = "foo"; Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + /* SDField fooRefToParent = new SDField( "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToParent); + */ var fooImports = fooSchema.temporaryImportedFields().get(); fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); @@ -40,9 +42,11 @@ public class ImportedFieldsEnumeratorTest { String BAR = "bar"; Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); + /* SDField barRefToParent = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(barRefToParent); + */ var barImports = barSchema.temporaryImportedFields().get(); barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java index c7ec2d9e9d1..652a06a6025 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java @@ -37,22 +37,22 @@ public class SDDocumentTypeOrdererTestCase { g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); - SDField aFieldTypeB = new TemporarySDField("atypeb", DataType.STRING); + SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); a.addField(aFieldTypeB); - SDField bFieldTypeC = new TemporarySDField("btypec", DataType.STRING); + SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); b.addField(bFieldTypeC); - SDField cFieldTypeG = new TemporarySDField("ctypeg", DataType.STRING); + SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); c.addField(cFieldTypeG); - SDField gFieldTypeF = new TemporarySDField("gtypef", DataType.STRING); + SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); g.addField(gFieldTypeF); - SDField fFieldTypeC = new TemporarySDField("ftypec", DataType.STRING); + SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); f.addField(fFieldTypeC); - SDField dFieldTypeE = new TemporarySDField("dtypee", DataType.STRING); + SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); d.addField(dFieldTypeE); types.add(a); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index 07c8025f27b..4b40da9e289 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -382,8 +382,8 @@ public class SchemaTestCase { " }" + " }" + "}"); - var application = ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile).application(); - var r3 = application.rankProfileRegistry().resolve(application.schemas().get("test").getDocument(), "r3"); + var builder = ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + var r3 = builder.getRankProfileRegistry().resolve(builder.application().schemas().get("test").getDocument(), "r3"); assertEquals(1, r3.allFilterFields().size()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index ad4ff4be4b3..e1cc42ba9c5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -28,7 +28,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { private static final String tempDir = "temp/"; private static final String searchDefRoot = "src/test/derived/"; - boolean useV8DocManagerCfg() { return false; } + boolean useV8DocManagerCfg() { return true; } private DerivedConfiguration derive(String dirName, String searchDefinitionName, diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java index 65d6c15d3e2..86895460dce 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java @@ -16,42 +16,56 @@ public class AnnotationsTestCase extends AbstractExportingTestCase { public void requireThatStructRegistersIfOnlyUsedByAnnotation() throws IOException, ParseException { assertCorrectDeriving("annotationsstruct", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationsstruct", + new TestProperties().setExperimentalSdParsing(false)); } @Test public void requireThatStructRegistersIfOnlyUsedAsArrayByAnnotation() throws IOException, ParseException { assertCorrectDeriving("annotationsstructarray", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationsstructarray", + new TestProperties().setExperimentalSdParsing(false)); } @Test public void testSimpleAnnotationDeriving() throws IOException, ParseException { assertCorrectDeriving("annotationssimple", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationssimple", + new TestProperties().setExperimentalSdParsing(false)); } @Test public void testAnnotationDerivingWithImplicitStruct() throws IOException, ParseException { assertCorrectDeriving("annotationsimplicitstruct", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationsimplicitstruct", + new TestProperties().setExperimentalSdParsing(false)); } @Test public void testAnnotationDerivingInheritance() throws IOException, ParseException { assertCorrectDeriving("annotationsinheritance", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationsinheritance", + new TestProperties().setExperimentalSdParsing(false)); } @Test public void testAnnotationDerivingInheritance2() throws IOException, ParseException { assertCorrectDeriving("annotationsinheritance2", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationsinheritance2", + new TestProperties().setExperimentalSdParsing(false)); } @Test public void testSimpleReference() throws IOException, ParseException { assertCorrectDeriving("annotationsreference", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationsreference", + new TestProperties().setExperimentalSdParsing(false)); } @Test @@ -64,8 +78,10 @@ public class AnnotationsTestCase extends AbstractExportingTestCase { public void testAnnotationsPolymorphy() throws IOException, ParseException { assertCorrectDeriving("annotationspolymorphy", new TestProperties().setExperimentalSdParsing(true)); + assertCorrectDeriving("annotationspolymorphy", + new TestProperties().setExperimentalSdParsing(false)); } - + /** * An annotation declared before document {} won't work, no doc type to add it to. */ @@ -83,5 +99,4 @@ public class AnnotationsTestCase extends AbstractExportingTestCase { assertCorrectDeriving("annotationsoutsideofdocument", new TestProperties().setExperimentalSdParsing(true)); } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 237e6c8c992..a3123550efa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -25,11 +25,11 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType doc = new SDDocumentType("test"); schema.addDocument(doc); - doc.addField(new SDField("a", DataType.STRING)); - SDField field = new SDField("b", DataType.STRING); + doc.addField(new SDField(doc, "a", DataType.STRING)); + SDField field = new SDField(doc, "b", DataType.STRING); field.setLiteralBoost(500); doc.addField(field); - doc.addField(new SDField("c", DataType.STRING)); + doc.addField(new SDField(doc, "c", DataType.STRING)); schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 440f067dd00..ba485b7b96b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -29,7 +29,7 @@ public class IdTestCase extends AbstractExportingTestCase { Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); - SDField uri = new SDField("URI", DataType.URI); + SDField uri = new SDField(document, "URI", DataType.URI); uri.parseIndexingScript("{ summary | index }"); document.addField(uri); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java index 5dcb9e4ca01..7559b193fba 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -25,6 +26,7 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase { } @Test + @Ignore public void configs_for_imported_struct_fields_are_derived_new() throws IOException, ParseException { assertCorrectDeriving("imported_struct_fields", "child", new TestProperties().setExperimentalSdParsing(true), diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index bcf68387294..10616731499 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.config.DocumentmanagerConfig; @@ -54,7 +55,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { try { assertCorrectDeriving("inheritfromnull"); } catch (IllegalArgumentException e) { - assertEquals("Document type 'foo' not found", e.getMessage()); + assertEquals("document inheritfromnull inherits from unavailable document foo", e.getMessage()); } } @@ -64,14 +65,17 @@ public class InheritanceTestCase extends AbstractExportingTestCase { List<String> files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); File outDir = tmpDir.newFolder("out"); for (int startIdx = 0; startIdx < files.size(); ++startIdx) { - ApplicationBuilder builder = new ApplicationBuilder(); + var builder = new ApplicationBuilder + (new TestProperties().setExperimentalSdParsing(true)); for (int fileIdx = startIdx; fileIdx < startIdx + files.size(); ++fileIdx) { String fileName = files.get(fileIdx % files.size()); builder.addSchemaFile(dir + fileName); } builder.build(true); DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); - DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); + DerivedConfiguration.exportDocuments(new DocumentManager(). + useV8DocManagerCfg(false). + produce(builder.getModel(), b), outDir.getPath()); DocumentmanagerConfig dc = b.build(); assertEquals(13, dc.datatype().size()); assertNull(structType("child.body", dc)); @@ -82,11 +86,11 @@ public class InheritanceTestCase extends AbstractExportingTestCase { assertEquals(childHeader.field(3).name(), "cox"); DocumentmanagerConfig.Datatype.Documenttype child = documentType("child", dc); assertEquals(child.inherits(0).name(), "document"); - assertEquals(child.inherits(1).name(), "father"); - assertEquals(child.inherits(2).name(), "mother"); + assertEquals(child.inherits(1).name(), "mother"); + assertEquals(child.inherits(2).name(), "father"); DocumentmanagerConfig.Datatype.Documenttype mother = documentType("mother", dc); - assertEquals(mother.inherits(0).name(), "grandparent"); - assertEquals(mother.inherits(1).name(), "document"); + assertEquals(mother.inherits(0).name(), "document"); + assertEquals(mother.inherits(1).name(), "grandparent"); } } @@ -163,4 +167,18 @@ public class InheritanceTestCase extends AbstractExportingTestCase { assertEquals(new Index("prefixed", true), childSchema.getIndex("prefixed")); } + @Test + public void testInheritStructDiamondNew() throws IOException, ParseException { + String dir = "src/test/derived/declstruct/"; + List<String> files = Arrays.asList("common.sd", "foo.sd", "bar.sd", "foobar.sd"); + var builder = new ApplicationBuilder + (new TestProperties().setExperimentalSdParsing(true)); + for (String fileName : files) { + builder.addSchemaFile(dir + fileName); + } + builder.build(true); + derive("declstruct", builder, builder.getSchema("foobar")); + assertCorrectConfigFiles("declstruct"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java new file mode 100755 index 00000000000..275c94465ae --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author arnej + */ +public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase { + + @Test + public void testSimpleDefaults() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_def", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_simple_def", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testSimpleWithStructSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_wss", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_simple_wss", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testSimpleWithFieldSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_wfs", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_simple_wfs", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testSimpleStructAndFieldSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_wss_wfs", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_simple_wss_wfs", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testMapDefaults() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_def", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_map_def", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testMapWithStructSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_wss", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_map_wss", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testMapWithFieldSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_wfs", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_map_wfs", + new TestProperties().setExperimentalSdParsing(true)); + } + + @Test + public void testMapAfter() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_after", + new TestProperties().setExperimentalSdParsing(false)); + assertCorrectDeriving("matchsettings_map_after", + new TestProperties().setExperimentalSdParsing(true)); + } + + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java new file mode 100644 index 00000000000..4f5316724f5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.searchdefinition.derived.TestableDeployLogger; +import com.yahoo.searchdefinition.ApplicationBuilder; +import org.junit.Test; + +/** + * Tests deriving a configuration with structs in multiple .sd files + * + * @author arnej + */ +public class MultiStructTestCase extends AbstractExportingTestCase { + + @Override + boolean useV8DocManagerCfg() { return false; } + + @Test + public void testDocTypeConfigs() throws Exception { + var logger = new TestableDeployLogger(); + var props = new TestProperties().setExperimentalSdParsing(false).setUseV8DocManagerCfg(false); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory + ("src/test/derived/multi_struct/", new MockFileRegistry(), logger, props); + derive("multi_struct", builder, builder.getSchema("shop")); + assertCorrectConfigFiles("multi_struct"); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java index 70f6187be12..61e80e6d701 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java @@ -19,21 +19,22 @@ public class NameCollisionTestCase extends AbstractExportingTestCase { @Test public void testNameCollision() throws Exception { - var ex = assertThrows(IllegalArgumentException.class, () -> { + var ex = assertThrows(RuntimeException.class, () -> { assertCorrectDeriving("namecollision", "collisionstruct", new TestProperties().setExperimentalSdParsing(false), new TestableDeployLogger()); var docman = DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); }); + ex.printStackTrace(); System.err.println("MSG 1: "+ex.getClass()+" -> "+ex.getMessage()); - var ey = assertThrows(IllegalArgumentException.class, () -> { + var ey = assertThrows(RuntimeException.class, () -> { assertCorrectDeriving("namecollision", "collisionstruct", new TestProperties().setExperimentalSdParsing(true), new TestableDeployLogger()); var docman = DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); }); - System.err.println("MSG 2: "+ey.getMessage()); - + ey.printStackTrace(); + System.err.println("MSG 2: "+ey.getClass()+" -> "+ey.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java new file mode 100644 index 00000000000..17684306cce --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.searchdefinition.derived.TestableDeployLogger; +import com.yahoo.searchdefinition.ApplicationBuilder; +import org.junit.Test; + +/** + * Tests deriving a configuration with references from multiple .sd files + * + * @author arnej + */ +public class ReferenceFromSeveralTestCase extends AbstractExportingTestCase { + + @Override + boolean useV8DocManagerCfg() { return false; } + + @Test + public void testDocManConfigs() throws Exception { + var logger = new TestableDeployLogger(); + var props = new TestProperties().setExperimentalSdParsing(false).setUseV8DocManagerCfg(false); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory + ("src/test/derived/reference_from_several/", new MockFileRegistry(), logger, props); + derive("reference_from_several", builder, builder.getSchema("foo")); + assertCorrectConfigFiles("reference_from_several"); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java index 34d33a00d9e..2522f8f56e2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java @@ -91,8 +91,8 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); fromDocument.addField(refField); Map<String, DocumentReference> originalMap = fromDocument.getDocumentReferences().get().referenceMap(); HashMap<String, DocumentReference> modifiedMap = new HashMap<>(originalMap); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java index 7989bff3152..1e4556c0de1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -14,6 +15,13 @@ import java.io.IOException; public class SortingTestCase extends AbstractExportingTestCase { @Test public void testDocumentDeriving() throws IOException, ParseException { - assertCorrectDeriving("sorting"); + assertCorrectDeriving("sorting", + new TestProperties().setExperimentalSdParsing(false)); + } + + @Test + public void testDocumentDerivingNewParser() throws IOException, ParseException { + assertCorrectDeriving("sorting", + new TestProperties().setExperimentalSdParsing(true)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index dbb32e61144..62a79e49146 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -30,7 +30,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); - SDField a = new SDField("a", DataType.STRING); + SDField a = new SDField(document, "a", DataType.STRING); a.parseIndexingScript("{ index }"); document.addField(a); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java index 5ab5a8057e8..9c974225605 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java @@ -25,8 +25,9 @@ public class VsmFieldsTestCase { @Test public void reference_type_field_is_unsearchable() { Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); - schema.addDocument(new SDDocumentType("test")); - SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); + var sdoc = new SDDocumentType("test"); + schema.addDocument(sdoc); + SDField refField = new TemporarySDField(sdoc, "ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); refField.parseIndexingScript("{ summary }"); schema.getDocument().addField(refField); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 9ca97f4dbc7..0adaba4cf68 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -59,8 +59,9 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { } private static ImportedFields createSingleImportedField(String fieldName) { - Schema targetSchema = createSearch("target_doc"); - SDField targetField = new SDField("target_field", DataType.INT); + Schema targetSchema = createSearchWithDocument("target_doc"); + var doc = targetSchema.getDocument(); + SDField targetField = new SDField(doc, "target_field", DataType.INT); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); return new ImportedFields(Collections.singletonMap(fieldName, importedField)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java index 5d2590a420d..926e26451d7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java @@ -186,9 +186,10 @@ public class AdjustPositionSummaryFieldsTestCase { private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) { String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }"; - schema.getDocument().addField(createField("pos", PositionDataType.INSTANCE, ilScript)); + var doc = schema.getDocument(); + doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript)); if (setupBadAttr) { - schema.getDocument().addField(createField("pos_zcurve", DataType.LONG, "{ attribute }")); + doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 324010f9e83..d1d3f4489ce 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -110,13 +110,14 @@ public class ImportedFieldsResolverTestCase { public SearchModel() { super(); grandParentSchema = createSearch("grandparent"); - grandParentSchema.getDocument().addField(createField("ancient_field", DataType.INT, "{ attribute }")); - - parentSchema.getDocument().addField(createField("attribute_field", DataType.INT, "{ attribute }")); - parentSchema.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }")); - parentSchema.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT)); - parentSchema.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); - parentSchema.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }")); + var grandParentDoc = grandParentSchema.getDocument(); + grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }")); + var parentDoc = parentSchema.getDocument(); + parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }")); + parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }")); + parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT)); + parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); + parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }")); addRefField(parentSchema, grandParentSchema, "ref"); addImportedField(parentSchema, "ancient_field", "ref", "ancient_field"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java index b14c7287537..3b4612ee87a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java @@ -37,19 +37,19 @@ public class ParentChildSearchModel { return result; } - protected static TemporarySDField createField(String name, DataType dataType, String indexingScript) { - TemporarySDField result = new TemporarySDField(name, dataType); + protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + TemporarySDField result = new TemporarySDField(repo, name, dataType); result.parseIndexingScript(indexingScript); return result; } @SuppressWarnings("deprecation") - protected static SDField createRefField(String parentType, String fieldName) { - return new TemporarySDField(fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType))); + protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) { + return new TemporarySDField(repo, fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType))); } protected static void addRefField(Schema child, Schema parent, String fieldName) { - SDField refField = createRefField(parent.getName(), fieldName); + SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName); child.getDocument().addField(refField); child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(), new DocumentReference(refField, parent)))); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java index d60cb9040d8..d353e070976 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.ReferenceDataType; @@ -57,7 +58,7 @@ public class ReferenceFieldTestCase { @Test public void cyclic_document_dependencies_are_detected() throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); + var builder = new ApplicationBuilder(new TestProperties().setExperimentalSdParsing(false)); String campaignSdContent = "search campaign {\n" + " document campaign {\n" + @@ -77,6 +78,28 @@ public class ReferenceFieldTestCase { builder.build(true); } + @Test + public void cyclic_document_dependencies_are_detected_new_parser() throws ParseException { + var builder = new ApplicationBuilder(new TestProperties().setExperimentalSdParsing(true)); + String campaignSdContent = + "search campaign {\n" + + " document campaign {\n" + + " field ad_ref type reference<ad> { indexing: attribute }\n" + + " }\n" + + "}"; + String adSdContent = + "search ad {\n" + + " document ad {\n" + + " field campaign_ref type reference<campaign> { indexing: attribute }\n" + + " }\n" + + "}"; + builder.addSchema(campaignSdContent); + builder.addSchema(adSdContent); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("reference cycle for documents"); + builder.build(true); + } + private static void assertSearchContainsReferenceField(String expectedFieldname, String referencedDocType, SDDocumentType documentType) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index 22fd4e45c4a..8c801d9deaf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -65,8 +65,8 @@ public class ValidateFieldTypesTest { } private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) { - Schema targetSchema = createSearch("target_doc"); - SDField targetField = new SDField("target_field", dataType); + Schema targetSchema = createSearchWithDocument("target_doc"); + SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); return new ImportedFields(Collections.singletonMap(fieldName, importedField)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index e58c29cc4fd..405c1127842 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -42,7 +42,7 @@ public class SchemaClusterTest { // sd1 SDDocumentType sdt1 = new SDDocumentType("s1"); Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); - SDField f1 = new SDField("f1", DataType.STRING); + SDField f1 = new SDField(sdt1, "f1", DataType.STRING); f1.addAttribute(new Attribute("f1", DataType.STRING)); f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); sdt1.addField(f1); @@ -51,7 +51,7 @@ public class SchemaClusterTest { // sd2 SDDocumentType sdt2 = new SDDocumentType("s2"); Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); - SDField f2=new SDField("f2", DataType.STRING); + SDField f2=new SDField(sdt2, "f2", DataType.STRING); f2.addAttribute(new Attribute("f2", DataType.STRING)); f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index a7f6a2a97a7..63122c0e0ec 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -109,7 +109,7 @@ case $1 in nohup nice sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS - jvmopts="-Xms32M -Xmx128M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" + jvmopts="-Xms32M -Xmx128M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=448 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" VESPA_SERVICE_NAME=configproxy export VESPA_SERVICE_NAME diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java index d72d720468c..b4d3a886b10 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java @@ -192,6 +192,8 @@ public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { } @Override + @Deprecated(since = "7", forRemoval = true) + @SuppressWarnings("removal") public void decode(InputStream is, Slime slime) throws IOException { throw new UnsupportedOperationException("decode is not supported"); } diff --git a/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java b/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java index 164fe36e2c6..6f0f121a9a6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java +++ b/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java @@ -38,31 +38,34 @@ public final class ErrorCode { public static final int INCONSISTENT_CONFIG_MD5 = UNKNOWN_CONFIG + 400; + public static final int INCOMPATIBLE_VESPA_VERSION = UNKNOWN_CONFIG + 500; + private ErrorCode() { } public static String getName(int error) { switch(error) { - case UNKNOWN_CONFIG: return "UNKNOWN_CONFIG"; - case UNKNOWN_DEFINITION: return "UNKNOWN_DEFINITION"; - case UNKNOWN_DEF_MD5: return "UNKNOWN_DEF_MD5"; - case ILLEGAL_NAME: return "ILLEGAL_NAME"; - case ILLEGAL_VERSION: return "ILLEGAL_VERSION"; - case ILLEGAL_CONFIGID: return "ILLEGAL_CONFIGID"; - case ILLEGAL_DEF_MD5: return "ILLEGAL_DEF_MD5"; - case ILLEGAL_CONFIG_MD5: return "ILLEGAL_CONFIG_MD5"; - case ILLEGAL_TIMEOUT: return "ILLEGAL_TIMEOUT"; - case ILLEGAL_GENERATION: return "ILLEGAL_GENERATION"; - case ILLEGAL_SUB_FLAG: return "ILLEGAL_SUBSCRIBE_FLAG"; - case ILLEGAL_NAME_SPACE: return "ILLEGAL_NAME_SPACE"; - case ILLEGAL_CLIENT_HOSTNAME: return "ILLEGAL_CLIENT_HOSTNAME"; - case OUTDATED_CONFIG: return "OUTDATED_CONFIG"; - case INTERNAL_ERROR: return "INTERNAL_ERROR"; - case APPLICATION_NOT_LOADED: return "APPLICATION_NOT_LOADED"; - case ILLEGAL_PROTOCOL_VERSION: return "ILLEGAL_PROTOCOL_VERSION"; - case INCONSISTENT_CONFIG_MD5: return "INCONSISTENT_CONFIG_MD5"; - case UNKNOWN_VESPA_VERSION: return "UNKNOWN_VESPA_VERSION"; - default: return "Unknown error"; + case UNKNOWN_CONFIG: return "UNKNOWN_CONFIG"; + case UNKNOWN_DEFINITION: return "UNKNOWN_DEFINITION"; + case UNKNOWN_DEF_MD5: return "UNKNOWN_DEF_MD5"; + case ILLEGAL_NAME: return "ILLEGAL_NAME"; + case ILLEGAL_VERSION: return "ILLEGAL_VERSION"; + case ILLEGAL_CONFIGID: return "ILLEGAL_CONFIGID"; + case ILLEGAL_DEF_MD5: return "ILLEGAL_DEF_MD5"; + case ILLEGAL_CONFIG_MD5: return "ILLEGAL_CONFIG_MD5"; + case ILLEGAL_TIMEOUT: return "ILLEGAL_TIMEOUT"; + case ILLEGAL_GENERATION: return "ILLEGAL_GENERATION"; + case ILLEGAL_SUB_FLAG: return "ILLEGAL_SUBSCRIBE_FLAG"; + case ILLEGAL_NAME_SPACE: return "ILLEGAL_NAME_SPACE"; + case ILLEGAL_CLIENT_HOSTNAME: return "ILLEGAL_CLIENT_HOSTNAME"; + case OUTDATED_CONFIG: return "OUTDATED_CONFIG"; + case INTERNAL_ERROR: return "INTERNAL_ERROR"; + case APPLICATION_NOT_LOADED: return "APPLICATION_NOT_LOADED"; + case ILLEGAL_PROTOCOL_VERSION: return "ILLEGAL_PROTOCOL_VERSION"; + case INCONSISTENT_CONFIG_MD5: return "INCONSISTENT_CONFIG_MD5"; + case UNKNOWN_VESPA_VERSION: return "UNKNOWN_VESPA_VERSION"; + case INCOMPATIBLE_VESPA_VERSION: return "INCOMPATIBLE_VESPA_VERSION"; + default: return "Unknown error"; } } diff --git a/configserver/pom.xml b/configserver/pom.xml index e81d1d5d970..d7702799dec 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -155,19 +155,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa_jersey2</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - <type>pom</type> - </dependency> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - <version>2.0</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> @@ -199,7 +186,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>serviceview</artifactId> <version>${project.version}</version> - <scope>compile</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -222,10 +208,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.glassfish.jersey.ext</groupId> - <artifactId>jersey-proxy-client</artifactId> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -234,8 +216,32 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> <version>${project.version}</version> - <scope>compile</scope> </dependency> + + <!-- Jersey, needed by serviceview --> + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <scope>provided</scope> <!-- TODO: Vespa 8: Set to compile if we get rid of the javax.ws.rs-api bundle --> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-client</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-server</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.ext</groupId> + <artifactId>jersey-proxy-client</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-json-jackson</artifactId> + </dependency> + <!-- Jersey END --> + </dependencies> <build> <plugins> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java index 99f5cce642a..20b16614a53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java @@ -92,4 +92,8 @@ public interface RequestHandler { * @return set of file references that is owned by the application */ Set<FileReference> listFileReferences(ApplicationId applicationId); + + /** Returns whether the latest deployed version of application is compatible with given vespaVersion */ + boolean compatibleWith(Optional<Version> vespaVersion, ApplicationId application); + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 9fbc3eb21df..0e4fffaccef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -108,6 +108,11 @@ public class SuperModelRequestHandler implements RequestHandler { throw new UnsupportedOperationException(); } + @Override + public boolean compatibleWith(Optional<Version> vespaVersion, ApplicationId application) { + return true; + } + public void enable() { enabled = true; superModelManager.markAsComplete(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 937ec2bb0e7..5e9e1e8bf33 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -26,6 +26,9 @@ import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.ListFlag; +import com.yahoo.vespa.flags.PermanentFlags; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -71,11 +74,12 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica private final Clock clock; private final TenantFileSystemDirs tenantFileSystemDirs; private final ConfigserverConfig configserverConfig; + private final ListFlag<Integer> incompatibleMajorVersions; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, ExecutorService zkCacheExecutor, Metrics metrics, ReloadListener reloadListener, ConfigserverConfig configserverConfig, HostRegistry hostRegistry, - TenantFileSystemDirs tenantFileSystemDirs, Clock clock) { + TenantFileSystemDirs tenantFileSystemDirs, Clock clock, FlagSource flagSource) { this.curator = curator; this.database = new ApplicationCuratorDatabase(tenant, curator); this.tenant = tenant; @@ -91,6 +95,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica this.tenantFileSystemDirs = tenantFileSystemDirs; this.clock = clock; this.configserverConfig = configserverConfig; + this.incompatibleMajorVersions = PermanentFlags.INCOMPATIBLE_MAJOR_VERSIONS.bindTo(flagSource); } /** The curator backed ZK storage of this. */ @@ -383,6 +388,15 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica } @Override + public boolean compatibleWith(Optional<Version> vespaVersion, ApplicationId application) { + if (vespaVersion.isEmpty()) return true; + Version wantedVersion = applicationMapper.getForVersion(application, Optional.empty(), clock.instant()) + .getModel().wantedNodeVersion(); + boolean compatibleMajor = ! incompatibleMajorVersions.value().contains(wantedVersion.getMajor()); + return compatibleMajor || vespaVersion.get().getMajor() == wantedVersion.getMajor(); + } + + @Override public void verifyHosts(ApplicationId applicationId, Collection<String> newHosts) { hostRegistry.verifyHosts(applicationId, newHosts); reloadListener.verifyHostsAreAvailable(applicationId, newHosts); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index dec5cbb7ee6..75ee9d66b4a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -212,54 +212,54 @@ public class ModelContextImpl implements ModelContext { private final boolean mergeGroupingResultInSearchInvoker; private final boolean experimentalSdParsing; - public FeatureFlags(FlagSource source, ApplicationId appId) { - this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); - this.useThreePhaseUpdates = flagValue(source, appId, Flags.USE_THREE_PHASE_UPDATES); - this.feedSequencer = flagValue(source, appId, Flags.FEED_SEQUENCER_TYPE); - this.feedTaskLimit = flagValue(source, appId, Flags.FEED_TASK_LIMIT); - this.feedMasterTaskLimit = flagValue(source, appId, Flags.FEED_MASTER_TASK_LIMIT); - this.sharedFieldWriterExecutor = flagValue(source, appId, Flags.SHARED_FIELD_WRITER_EXECUTOR); - this.responseSequencer = flagValue(source, appId, Flags.RESPONSE_SEQUENCER_TYPE); - this.numResponseThreads = flagValue(source, appId, Flags.RESPONSE_NUM_THREADS); - this.skipCommunicationManagerThread = flagValue(source, appId, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); - this.skipMbusRequestThread = flagValue(source, appId, Flags.SKIP_MBUS_REQUEST_THREAD); - this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); - this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); - this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); - this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); - this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); - this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); - this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); - this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); - this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); - this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); - this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); - this.metricsproxyNumThreads = flagValue(source, appId, Flags.METRICSPROXY_NUM_THREADS); - this.availableProcessors = flagValue(source, appId, Flags.AVAILABLE_PROCESSORS); - this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); - this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); - this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY); - this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); - this.ignoreThreadStackSizes = flagValue(source, appId, Flags.IGNORE_THREAD_STACK_SIZES); - this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); - this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); - this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); - this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); - this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); - this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); - this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); - this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); - this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); - this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); - this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); - this.persistenceThrottlingWindowSize = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WINDOW_SIZE); - this.persistenceThrottlingWsResizeRate = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_RESIZE_RATE); - this.persistenceThrottlingOfMergeFeedOps = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_OF_MERGE_FEED_OPS); - this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); - this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); - this.avoidRenamingSummaryFeatures = flagValue(source, appId, Flags.AVOID_RENAMING_SUMMARY_FEATURES); - this.mergeGroupingResultInSearchInvoker = flagValue(source, appId, Flags.MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER); - this.experimentalSdParsing = flagValue(source, appId, Flags.EXPERIMENTAL_SD_PARSING); + public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { + this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); + this.useThreePhaseUpdates = flagValue(source, appId, version, Flags.USE_THREE_PHASE_UPDATES); + this.feedSequencer = flagValue(source, appId, version, Flags.FEED_SEQUENCER_TYPE); + this.feedTaskLimit = flagValue(source, appId, version, Flags.FEED_TASK_LIMIT); + this.feedMasterTaskLimit = flagValue(source, appId, version, Flags.FEED_MASTER_TASK_LIMIT); + this.sharedFieldWriterExecutor = flagValue(source, appId, version, Flags.SHARED_FIELD_WRITER_EXECUTOR); + this.responseSequencer = flagValue(source, appId, version, Flags.RESPONSE_SEQUENCER_TYPE); + this.numResponseThreads = flagValue(source, appId, version, Flags.RESPONSE_NUM_THREADS); + this.skipCommunicationManagerThread = flagValue(source, appId, version, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); + this.skipMbusRequestThread = flagValue(source, appId, version, Flags.SKIP_MBUS_REQUEST_THREAD); + this.skipMbusReplyThread = flagValue(source, appId, version, Flags.SKIP_MBUS_REPLY_THREAD); + this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, version, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); + this.feedConcurrency = flagValue(source, appId, version, Flags.FEED_CONCURRENCY); + this.allowedAthenzProxyIdentities = flagValue(source, appId, version, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); + this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, version, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); + this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); + this.maxConcurrentMergesPerContentNode = flagValue(source, appId, version, Flags.MAX_CONCURRENT_MERGES_PER_NODE); + this.maxMergeQueueSize = flagValue(source, appId, version, Flags.MAX_MERGE_QUEUE_SIZE); + this.resourceLimitDisk = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_DISK); + this.resourceLimitMemory = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_MEMORY); + this.minNodeRatioPerGroup = flagValue(source, appId, version, Flags.MIN_NODE_RATIO_PER_GROUP); + this.metricsproxyNumThreads = flagValue(source, appId, version, Flags.METRICSPROXY_NUM_THREADS); + this.availableProcessors = flagValue(source, appId, version, Flags.AVAILABLE_PROCESSORS); + this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, version, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); + this.containerShutdownTimeout = flagValue(source, appId, version, Flags.CONTAINER_SHUTDOWN_TIMEOUT); + this.maxUnCommittedMemory = flagValue(source, appId, version, Flags.MAX_UNCOMMITTED_MEMORY); + this.forwardIssuesAsErrors = flagValue(source, appId, version, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); + this.ignoreThreadStackSizes = flagValue(source, appId, version, Flags.IGNORE_THREAD_STACK_SIZES); + this.unorderedMergeChaining = flagValue(source, appId, version, Flags.UNORDERED_MERGE_CHAINING); + this.useV8GeoPositions = flagValue(source, appId, version, Flags.USE_V8_GEO_POSITIONS); + this.useV8DocManagerCfg = flagValue(source, appId, version, Flags.USE_V8_DOC_MANAGER_CFG); + this.maxCompactBuffers = flagValue(source, appId, version, Flags.MAX_COMPACT_BUFFERS); + this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, version, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); + this.ignoredHttpUserAgents = flagValue(source, appId, version, PermanentFlags.IGNORED_HTTP_USER_AGENTS); + this.enableServerOcspStapling = flagValue(source, appId, version, Flags.ENABLE_SERVER_OCSP_STAPLING); + this.persistenceAsyncThrottling = flagValue(source, appId, version, Flags.PERSISTENCE_ASYNC_THROTTLING); + this.mergeThrottlingPolicy = flagValue(source, appId, version, Flags.MERGE_THROTTLING_POLICY); + this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); + this.persistenceThrottlingWsBackoff = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); + this.persistenceThrottlingWindowSize = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WINDOW_SIZE); + this.persistenceThrottlingWsResizeRate = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_WS_RESIZE_RATE); + this.persistenceThrottlingOfMergeFeedOps = flagValue(source, appId, version, Flags.PERSISTENCE_THROTTLING_OF_MERGE_FEED_OPS); + this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, version, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); + this.useQrserverServiceName = flagValue(source, appId, version, Flags.USE_QRSERVER_SERVICE_NAME); + this.avoidRenamingSummaryFeatures = flagValue(source, appId, version, Flags.AVOID_RENAMING_SUMMARY_FEATURES); + this.mergeGroupingResultInSearchInvoker = flagValue(source, appId, version, Flags.MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER); + this.experimentalSdParsing = flagValue(source, appId, version, Flags.EXPERIMENTAL_SD_PARSING); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -312,33 +312,38 @@ public class ModelContextImpl implements ModelContext { @Override public boolean mergeGroupingResultInSearchInvoker() { return mergeGroupingResultInSearchInvoker; } @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } - private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { + private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()) + .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); } - private static <V> V flagValue(FlagSource source, TenantName tenant, UnboundFlag<? extends V, ?, ?> flag) { + private static <V> V flagValue(FlagSource source, TenantName tenant, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.TENANT_ID, tenant.value()) + .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); } private static <V> V flagValue(FlagSource source, ApplicationId appId, + Version vespaVersion, ClusterSpec.Type clusterType, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()) .with(FetchVector.Dimension.CLUSTER_TYPE, clusterType.name()) + .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); } static int flagValueAsInt(FlagSource source, ApplicationId appId, + Version version, ClusterSpec.Type clusterType, UnboundFlag<? extends Boolean, ?, ?> flag) { - return flagValue(source, appId, clusterType, flag) ? 1 : 0; + return flagValue(source, appId, version, clusterType, flag) ? 1 : 0; } private String translateJvmOmitStackTraceInFastThrowIntToString(ToIntFunction<ClusterSpec.Type> function, @@ -375,6 +380,7 @@ public class ModelContextImpl implements ModelContext { private final List<String> environmentVariables; public Properties(ApplicationId applicationId, + Version nodeVespaVersion, ConfigserverConfig configserverConfig, Zone zone, Set<ContainerEndpoint> endpoints, @@ -387,7 +393,7 @@ public class ModelContextImpl implements ModelContext { List<TenantSecretStore> tenantSecretStores, SecretStore secretStore, List<X509Certificate> operatorCertificates) { - this.featureFlags = new FeatureFlags(flagSource, applicationId); + this.featureFlags = new FeatureFlags(flagSource, applicationId, nodeVespaVersion); this.applicationId = applicationId; this.multitenant = configserverConfig.multitenant() || configserverConfig.hostedVespa() || Boolean.getBoolean("multitenant"); this.configServerSpecs = fromConfig(configserverConfig); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 54ceb394ee6..f2d9eb835be 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -102,7 +102,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ) { log.log(Level.FINE, () -> String.format("Loading model version %s for session %s application %s", modelFactory.version(), applicationGeneration, applicationId)); - ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, applicationPackage); + ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, wantedNodeVespaVersion, applicationPackage); Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, @@ -146,8 +146,9 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { return Optional.of(value); } - private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, ApplicationPackage applicationPackage) { + private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, Version wantedNodeVespaVersion, ApplicationPackage applicationPackage) { return new ModelContextImpl.Properties(applicationId, + wantedNodeVespaVersion, configserverConfig, zone(), ImmutableSet.copyOf(new ContainerEndpointsCache(TenantRepository.getTenantPath(tenant), curator).read(applicationId)), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 88a30a9d72b..55645d83b3c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -100,11 +100,16 @@ class GetConfigProcessor implements Runnable { request.getVespaVersion().map(VespaVersion::toString).map(Version::fromString) : Optional.empty(); if (logDebug(trace)) { - debugLog(trace, "Using version " + getPrintableVespaVersion(vespaVersion)); + debugLog(trace, "Using version " + printableVespaVersion(vespaVersion)); } if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) { - handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + getPrintableVespaVersion(vespaVersion)); + handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + printableVespaVersion(vespaVersion)); + return null; + } + + if ( ! context.requestHandler().compatibleWith(vespaVersion, context.applicationId())) { + handleError(request, ErrorCode.INCOMPATIBLE_VESPA_VERSION, "Version " + printableVespaVersion(vespaVersion) + " is binary incompatible with the latest deployed version"); return null; } @@ -161,7 +166,7 @@ class GetConfigProcessor implements Runnable { request.getConfigKey().getNamespace().equals(SentinelConfig.getDefNamespace()); } - private static String getPrintableVespaVersion(Optional<Version> vespaVersion) { + private static String printableVespaVersion(Optional<Version> vespaVersion) { return (vespaVersion.isPresent() ? vespaVersion.get().toFullString() : "LATEST"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index f8ee3e5e0c9..8293871335d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -191,6 +191,7 @@ public class SessionPreparer { this.containerEndpoints = readEndpointsIfNull(params.containerEndpoints()); this.athenzDomain = params.athenzDomain(); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), + vespaVersion, configserverConfig, zone, Set.copyOf(containerEndpoints), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 7d53c383cf5..82d8a42bc92 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -342,7 +342,8 @@ public class TenantRepository { configserverConfig, hostRegistry, new TenantFileSystemDirs(configServerDB, tenantName), - clock); + clock, + flagSource); PermanentApplicationPackage permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionFactory, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 659bc771ac7..41e6fe98441 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -63,6 +63,7 @@ public class ModelContextImplTest { new Provisioned(), new ModelContextImpl.Properties( ApplicationId.defaultId(), + Version.emptyVersion, configserverConfig, Zone.defaultZone(), endpoints, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 5afcd62a25c..c95c95750a1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -3,10 +3,12 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; @@ -26,6 +28,8 @@ import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.curator.mock.MockCuratorFramework; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; import org.apache.curator.framework.CuratorFramework; @@ -47,6 +51,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.IntStream; import static com.yahoo.vespa.config.server.application.TenantApplications.RemoveApplicationWaiter; @@ -66,7 +72,6 @@ public class TenantApplicationsTest { private Curator curator; private CuratorFramework curatorFramework; - private TenantApplications applications; private ConfigserverConfig configserverConfig; @Rule @@ -88,7 +93,6 @@ public class TenantApplicationsTest { .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenantName); - applications = createTenantApplications(tenantName, curator, configserverConfig, new MockReloadListener()); } @Test @@ -150,6 +154,47 @@ public class TenantApplicationsTest { assertEquals(0, repo.activeApplications().size()); } + private static ApplicationSet createSet(ApplicationId id, Version version) throws IOException, SAXException { + VespaModel model = new VespaModel(new NullConfigModelRegistry(), + new DeployState.Builder().wantedNodeVespaVersion(version) + .applicationPackage(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))) + .build()); + return ApplicationSet.from(new Application(model, + new ServerCache(), + 1, + Version.emptyVersion, + MetricUpdater.createTestUpdater(), + id)); + } + + @Test + public void major_version_compatibility() throws Exception { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); + TenantApplications applications = createZKAppRepo(flagSource); + ApplicationId app1 = createApplicationId("myapp"); + applications.createApplication(app1); + applications.createPutTransaction(app1, 1).commit(); + + Version deployedVersion0 = Version.fromString("6.1"); + applications.activateApplication(createSet(app1, deployedVersion0), 1); + assertTrue("Empty version is compatible", applications.compatibleWith(Optional.empty(), app1)); + + Version nodeVersion0 = Version.fromString("6.0"); + assertTrue("Lower version is compatible", applications.compatibleWith(Optional.of(nodeVersion0), app1)); + + Version deployedVersion1 = Version.fromString("7.1"); + applications.activateApplication(createSet(app1, deployedVersion1), 1); + assertTrue("New major is compatible", applications.compatibleWith(Optional.of(nodeVersion0), app1)); + + flagSource.withListFlag(PermanentFlags.INCOMPATIBLE_MAJOR_VERSIONS.id(), List.of(8), Integer.class); + Version deployedVersion2 = Version.fromString("8.1"); + applications.activateApplication(createSet(app1, deployedVersion2), 1); + assertFalse("New major is incompatible", applications.compatibleWith(Optional.of(nodeVersion0), app1)); + + Version nodeVersion1 = Version.fromString("8.0"); + assertTrue("Node is compatible after upgrading", applications.compatibleWith(Optional.of(nodeVersion1), app1)); + } + public static class MockReloadListener implements ReloadListener { public final AtomicInteger reloaded = new AtomicInteger(0); final AtomicInteger removed = new AtomicInteger(0); @@ -175,14 +220,10 @@ public class TenantApplicationsTest { } } - private void assertdefaultAppNotFound() { - assertFalse(applications.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); - } - @Test public void testListConfigs() throws IOException, SAXException { - applications = createTenantApplications(TenantName.defaultName(), new MockCurator(), configserverConfig, new MockReloadListener()); - assertdefaultAppNotFound(); + TenantApplications applications = createTenantApplications(TenantName.defaultName(), new MockCurator(), configserverConfig, new MockReloadListener(), new InMemoryFlagSource()); + assertFalse(applications.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); ApplicationId applicationId = ApplicationId.defaultId(); @@ -209,6 +250,7 @@ public class TenantApplicationsTest { @Test public void testAppendIdsInNonRecursiveListing() { + TenantApplications applications = createTenantApplications(tenantName, curator, configserverConfig, new MockReloadListener(), new InMemoryFlagSource()); assertEquals(applications.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); assertEquals(applications.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); assertEquals(applications.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); @@ -251,9 +293,12 @@ public class TenantApplicationsTest { } private TenantApplications createZKAppRepo() { - return createTenantApplications(tenantName, curator, configserverConfig, new MockReloadListener()); + return createZKAppRepo(new InMemoryFlagSource()); } + private TenantApplications createZKAppRepo(InMemoryFlagSource flagSource) { + return createTenantApplications(tenantName, curator, configserverConfig, new MockReloadListener(), flagSource); + } private static ApplicationId createApplicationId() { return createApplicationId("foo"); @@ -285,17 +330,18 @@ public class TenantApplicationsTest { private TenantApplications createTenantApplications(TenantName tenantName, Curator curator, ConfigserverConfig configserverConfig, - ReloadListener reloadListener) { + ReloadListener reloadListener, InMemoryFlagSource flagSource) { return new TenantApplications(tenantName, - curator, - new StripedExecutor<>(new InThreadExecutorService()), - new InThreadExecutorService(), - Metrics.createTestMetrics(), - reloadListener, - configserverConfig, - new HostRegistry(), - new TenantFileSystemDirs(new ConfigServerDB(configserverConfig), tenantName), - Clock.systemUTC()); + curator, + new StripedExecutor<>(new InThreadExecutorService()), + new InThreadExecutorService(), + Metrics.createTestMetrics(), + reloadListener, + configserverConfig, + new HostRegistry(), + new TenantFileSystemDirs(new ConfigServerDB(configserverConfig), tenantName), + Clock.systemUTC(), + flagSource); } private static class MockCurator3ConfigServers extends Curator { diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 9cc519ed9e2..0e45aa59421 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -4121,6 +4121,19 @@ ], "fields": [] }, + "com.yahoo.search.grouping.result.FlatteningSearcher": { + "superClass": "com.yahoo.search.Searcher", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)", + "public void flatten(com.yahoo.search.result.HitGroup, com.yahoo.search.Result)" + ], + "fields": [] + }, "com.yahoo.search.grouping.result.Group": { "superClass": "com.yahoo.search.result.HitGroup", "interfaces": [], diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 7f411bbc80f..27a45753bb5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -81,7 +81,7 @@ public class FastSearcher extends VespaBackEndSearcher { @Override public Result doSearch2(Query query, Execution execution) { - if (dispatcher.searchCluster().wantedGroupSize() == 1) + if (dispatcher.searchCluster().allGroupsHaveSize1()) forceSinglePassGrouping(query); try (SearchInvoker invoker = getSearchInvoker(query)) { Result result = invoker.search(query, execution); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java index d017cce0d44..3c61a361cbb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java @@ -13,9 +13,8 @@ import com.yahoo.search.searchchain.Execution; import java.util.*; /** - * <p> Groups hits according to sddocname. </p> - * - * <p> For each group, the desired number of hits can be specified. </p> + * Groups hits according to document type. + * For each group, the desired number of hits can be specified. * * @author Tony Vaagenes */ @@ -48,7 +47,7 @@ public class MultipleResultsSearcher extends Searcher { } } - private class HitsRetriever { + private static class HitsRetriever { PartitionedResult partitionedResult; @@ -58,12 +57,12 @@ public class MultipleResultsSearcher extends Searcher { private final Parameters parameters; private final int hits; private final int offset; - private Execution execution; - private Result initialResult; + private final Execution execution; + private final Result initialResult; HitsRetriever(Query query, Execution execution, Parameters parameters) throws ParameterException { - this.offset=query.getOffset(); - this.hits=query.getHits(); + this.offset = query.getOffset(); + this.hits = query.getHits(); this.nextOffset = query.getOffset() + query.getHits(); this.query = query; this.parameters = parameters; @@ -362,13 +361,14 @@ public class MultipleResultsSearcher extends Searcher { } } - @SuppressWarnings("serial") private static class ParameterException extends Exception { + String msg; ParameterException(String msg) { this.msg = msg; } + } } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index fb7281e1f24..83fa18d847f 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -726,8 +726,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * if the trace level of the query is sufficiently high. * * @param message the message to add - * @param includeQuery true to append the query root stringValue - * at the end of the message + * @param includeQuery true to append the query root stringValue at the end of the message * @param traceLevel the context level of the message, this method will do nothing * if the traceLevel of the query is lower than this value */ diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index d7c9f1dce53..5e38f6b4bdd 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -116,8 +116,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM InvokerResult result = new InvokerResult(query, query.getHits()); List<LeanHit> merged = Collections.emptyList(); long nextTimeout = query.getTimeLeft(); - boolean extraDebug = (query.getOffset() == 0) && (query.getHits() == 7) && log.isLoggable(java.util.logging.Level.FINE); - List<InvokerResult> processed = new ArrayList<>(); var groupingResultAggregator = new GroupingResultAggregator(); try { while (!invokers.isEmpty() && nextTimeout >= 0) { @@ -127,9 +125,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM break; } else { InvokerResult toMerge = invoker.getSearchResult(execution); - if (extraDebug) { - processed.add(toMerge); - } merged = mergeResult(result.getResult(), toMerge, merged, groupingResultAggregator); ejectInvoker(invoker); } @@ -143,32 +138,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM insertNetworkErrors(result.getResult()); result.getResult().setCoverage(createCoverage()); - if (extraDebug && merged.size() > 0) { - int firstPartId = merged.get(0).getPartId(); - for (int index = 1; index < merged.size(); index++) { - if (merged.get(index).getPartId() != firstPartId) { - extraDebug = false; - log.fine("merged["+index+"/"+merged.size()+"] from partId "+merged.get(index).getPartId()+", first "+firstPartId); - break; - } - } - } - if (extraDebug) { - log.fine("Interleaved "+processed.size()+" results"); - for (int pIdx = 0; pIdx < processed.size(); ++pIdx) { - var p = processed.get(pIdx); - log.fine("InvokerResult "+pIdx+" total hits "+p.getResult().getTotalHitCount()); - var lean = p.getLeanHits(); - for (int idx = 0; idx < lean.size(); ++idx) { - var hit = lean.get(idx); - log.fine("lean hit "+idx+" relevance "+hit.getRelevance()+" partid "+hit.getPartId()); - } - } - for (int mIdx = 0; mIdx < merged.size(); ++mIdx) { - var hit = merged.get(mIdx); - log.fine("merged hit "+mIdx+" relevance "+hit.getRelevance()+" partid "+hit.getPartId()); - } - } int needed = query.getOffset() + query.getHits(); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { result.getLeanHits().add(merged.get(index)); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 7f4d8fc4739..4c0bcb38d15 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -12,13 +12,14 @@ import java.util.Set; import java.util.logging.Logger; /** - * LoadBalancer determines which group of content nodes should be accessed next for each search query when the internal java dispatcher is - * used. + * LoadBalancer determines which group of content nodes should be accessed next for each search query when the + * internal java dispatcher is used. + * + * The implementation here is a simplistic least queries in flight + round-robin load balancer * * @author ollivir */ public class LoadBalancer { - // The implementation here is a simplistic least queries in flight + round-robin load balancer private static final Logger log = Logger.getLogger(LoadBalancer.class.getName()); @@ -84,6 +85,7 @@ public class LoadBalancer { } static class GroupStatus { + private final Group group; private int allocations = 0; private long queries = 0; @@ -174,24 +176,10 @@ public class LoadBalancer { * @return the better of the two */ private static GroupStatus betterGroup(GroupStatus first, GroupStatus second) { - if (second == null) { - return first; - } - if (first == null) { - return second; - } - - // different coverage - if (first.group.hasSufficientCoverage() != second.group.hasSufficientCoverage()) { - if (!first.group.hasSufficientCoverage()) { - // first doesn't have coverage, second does - return second; - } else { - // second doesn't have coverage, first does - return first; - } - } - + if (second == null) return first; + if (first == null) return second; + if (first.group.hasSufficientCoverage() != second.group.hasSufficientCoverage()) + return first.group.hasSufficientCoverage() ? first : second; return first; } @@ -246,11 +234,8 @@ public class LoadBalancer { public Optional<GroupStatus> takeNextGroup(Set<Integer> rejectedGroups) { double needle = random.nextDouble(); Optional<GroupStatus> gs = selectGroup(needle, true, rejectedGroups); - if (gs.isPresent()) { - return gs; - } - // fallback - any coverage better than none - return selectGroup(needle, false, rejectedGroups); + if (gs.isPresent()) return gs; + return selectGroup(needle, false, rejectedGroups); // any coverage better than none } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 5e12fb550c8..af9834e282a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -17,6 +17,7 @@ import java.util.logging.Logger; public class Group { private static final Logger log = Logger.getLogger(Group.class.getName()); + private final static double maxContentSkew = 0.10; private final static int minDocsPerNodeToRequireLowSkew = 100; @@ -41,8 +42,7 @@ public class Group { /** * Returns the unique identity of this group. - * NOTE: This is a contiguous index from 0, NOT necessarily the group id assigned - * by the user or node repo. + * NOTE: This is a contiguous index from 0, NOT necessarily the group id assigned by the user or node repo. */ public int id() { return id; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 36d7e7a85a9..f8c4627473d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; import com.google.common.math.Quantiles; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; @@ -32,11 +29,10 @@ public class SearchCluster implements NodeManager<Node> { private static final Logger log = Logger.getLogger(SearchCluster.class.getName()); private final DispatchConfig dispatchConfig; - private final int size; private final String clusterId; - private final ImmutableMap<Integer, Group> groups; - private final ImmutableMultimap<String, Node> nodesByHost; - private final ImmutableList<Group> orderedGroups; + private final Map<Integer, Group> groups; + private final List<Group> orderedGroups; + private final List<Node> nodes; private final VipStatus vipStatus; private final PingFactory pingFactory; private final TopKEstimator hitEstimator; @@ -60,8 +56,7 @@ public class SearchCluster implements NodeManager<Node> { this.vipStatus = vipStatus; this.pingFactory = pingFactory; - List<Node> nodes = toNodes(dispatchConfig); - this.size = nodes.size(); + this.nodes = toNodes(dispatchConfig); // Create groups ImmutableMap.Builder<Integer, Group> groupsBuilder = new ImmutableMap.Builder<>(); @@ -72,16 +67,10 @@ public class SearchCluster implements NodeManager<Node> { this.groups = groupsBuilder.build(); LinkedHashMap<Integer, Group> groupIntroductionOrder = new LinkedHashMap<>(); nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group()))); - this.orderedGroups = ImmutableList.<Group>builder().addAll(groupIntroductionOrder.values()).build(); + this.orderedGroups = List.copyOf(groupIntroductionOrder.values()); - // Index nodes by host - ImmutableMultimap.Builder<String, Node> nodesByHostBuilder = new ImmutableMultimap.Builder<>(); - for (Node node : nodes) - nodesByHostBuilder.put(node.hostname(), node); - this.nodesByHost = nodesByHostBuilder.build(); hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); - - this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), nodesByHost, groups); + this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), nodes, groups); } @Override @@ -95,13 +84,15 @@ public class SearchCluster implements NodeManager<Node> { } private static Optional<Node> findLocalCorpusDispatchTarget(String selfHostname, - ImmutableMultimap<String, Node> nodesByHost, - ImmutableMap<Integer, Group> groups) { + List<Node> nodes, + Map<Integer, Group> groups) { // A search node in the search cluster in question is configured on the same host as the currently running container. // It has all the data <==> No other nodes in the search cluster have the same group id as this node. // That local search node responds. // The search cluster to be searched has at least as many nodes as the container cluster we're running in. - ImmutableCollection<Node> localSearchNodes = nodesByHost.get(selfHostname); + List<Node> localSearchNodes = nodes.stream() + .filter(node -> node.hostname().equals(selfHostname)) + .collect(Collectors.toList()); // Only use direct dispatch if we have exactly 1 search node on the same machine: if (localSearchNodes.size() != 1) return Optional.empty(); @@ -114,25 +105,24 @@ public class SearchCluster implements NodeManager<Node> { return Optional.of(localSearchNode); } - private static ImmutableList<Node> toNodes(DispatchConfig dispatchConfig) { - ImmutableList.Builder<Node> nodesBuilder = new ImmutableList.Builder<>(); - for (DispatchConfig.Node node : dispatchConfig.node()) - nodesBuilder.add(new Node(node.key(), node.host(), node.group())); - return nodesBuilder.build(); + private static List<Node> toNodes(DispatchConfig dispatchConfig) { + return dispatchConfig.node().stream() + .map(n -> new Node(n.key(), n.host(), n.group())) + .collect(Collectors.toUnmodifiableList()); } public DispatchConfig dispatchConfig() { return dispatchConfig; } - /** Returns the number of nodes in this cluster (across all groups) */ - public int size() { return size; } + /** Returns an immutable list of all nodes in this. */ + public List<Node> nodes() { return nodes; } /** Returns the groups of this cluster as an immutable map indexed by group id */ - public ImmutableMap<Integer, Group> groups() { return groups; } + public Map<Integer, Group> groups() { return groups; } /** Returns the groups of this cluster as an immutable list in introduction order */ - public ImmutableList<Group> orderedGroups() { return orderedGroups; } + public List<Group> orderedGroups() { return orderedGroups; } /** Returns the n'th (zero-indexed) group in the cluster if possible */ public Optional<Group> group(int n) { @@ -143,23 +133,12 @@ public class SearchCluster implements NodeManager<Node> { } } - /** - * Returns the wanted number of nodes per group - size()/groups.size(). - * The actual node count for a given group may differ due to node retirements. - */ - public int wantedGroupSize() { - if (groups().size() == 0) return size(); - return size() / groups().size(); + public boolean allGroupsHaveSize1() { + return nodes.size() == groups.size(); } public int groupsWithSufficientCoverage() { - int covered = 0; - for (Group g : orderedGroups()) { - if (g.hasSufficientCoverage()) { - covered++; - } - } - return covered; + return (int)groups.values().stream().filter(g -> g.hasSufficientCoverage()).count(); } /** @@ -210,8 +189,8 @@ public class SearchCluster implements NodeManager<Node> { } else if (usesLocalCorpusIn(node)) { // follow the status of this node // Do not take this out of rotation if we're a combined cluster of size 1, - // as that can't be helpful, and leads to a deadlock where this node is never taken back in servic e - if (nodeIsWorking || size() > 1) + // as that can't be helpful, and leads to a deadlock where this node is never set back in service + if (nodeIsWorking || nodes.size() > 1) setInRotationOnlyIf(nodeIsWorking); } } @@ -240,11 +219,11 @@ public class SearchCluster implements NodeManager<Node> { } public boolean hasInformationAboutAllNodes() { - return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null); + return nodes.stream().allMatch(node -> node.isWorking() != null); } private boolean hasWorkingNodes() { - return nodesByHost.values().stream().anyMatch(node -> node.isWorking() != Boolean.FALSE ); + return nodes.stream().anyMatch(node -> node.isWorking() != Boolean.FALSE ); } private boolean usesLocalCorpusIn(Node node) { @@ -255,31 +234,6 @@ public class SearchCluster implements NodeManager<Node> { return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id(); } - private static class PongCallback implements PongHandler { - - private final ClusterMonitor<Node> clusterMonitor; - private final Node node; - - PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) { - this.node = node; - this.clusterMonitor = clusterMonitor; - } - - @Override - public void handle(Pong pong) { - if (pong.badResponse()) { - clusterMonitor.failed(node, pong.error().get()); - } else { - if (pong.activeDocuments().isPresent()) { - node.setActiveDocuments(pong.activeDocuments().get()); - node.setBlockingWrites(pong.isBlockingWrites()); - } - clusterMonitor.responded(node); - } - } - - } - /** Used by the cluster monitor to manage node status */ @Override public void ping(ClusterMonitor clusterMonitor, Node node, Executor executor) { @@ -293,19 +247,15 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), - group.activeDocuments()); + boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), group.activeDocuments()); trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments()); } private void pingIterationCompletedMultipleGroups() { orderedGroups().forEach(Group::aggregateNodeValues); long medianDocuments = medianDocumentsPerGroup(); - boolean anyGroupsSufficientCoverage = false; for (Group group : orderedGroups()) { - boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), - medianDocuments); - anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; + boolean sufficientCoverage = isGroupCoverageSufficient(group.activeDocuments(), medianDocuments); updateSufficientCoverage(group, sufficientCoverage); trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); } @@ -372,4 +322,29 @@ public class SearchCluster implements NodeManager<Node> { } } + private static class PongCallback implements PongHandler { + + private final ClusterMonitor<Node> clusterMonitor; + private final Node node; + + PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) { + this.node = node; + this.clusterMonitor = clusterMonitor; + } + + @Override + public void handle(Pong pong) { + if (pong.badResponse()) { + clusterMonitor.failed(node, pong.error().get()); + } else { + if (pong.activeDocuments().isPresent()) { + node.setActiveDocuments(pong.activeDocuments().get()); + node.setBlockingWrites(pong.isBlockingWrites()); + } + clusterMonitor.responded(node); + } + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java new file mode 100644 index 00000000000..321f86facd0 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +import java.util.Iterator; + +/** + * Flattens a grouping result into a flat list of hits on the top level in the returned result. + * Useful when using grouping to create hits with diversity and similar. + * + * @author bratseth + */ +@Before(GroupingExecutor.COMPONENT_NAME) +public class FlatteningSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + if ( ! query.properties().getBoolean("flatten", true)) return execution.search(query); + + query.trace("Flattening groups", 2); + int originalHits = query.getHits(); + query.setHits(0); + Result result = execution.search(query); + query.setHits(originalHits); + flatten(result.hits(), result); + return result; + } + + public void flatten(HitGroup hits, Result result) { + int hitsLeft = hits.size(); // Iterate only through the initial size + for (Iterator<Hit> i = hits.iterator(); i.hasNext() && hitsLeft-- > 0;) { + Hit hit = i.next(); + if (hit instanceof HitGroup) { + flatten((HitGroup)hit, result); + i.remove(); + } else { + result.hits().add(hit); + } + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 6d09bf66175..efe25e04f2e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -249,9 +249,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< return hit; } - /** - * Adds a list of hits to this group, the same - */ + /** Adds a list of hits to this group, the same as calling add for each item in the list. */ public void addAll(List<Hit> hits) { for (Hit hit : hits) add(hit); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 54c8c1e0522..abd7267bb04 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -23,7 +23,7 @@ public class MockSearchCluster extends SearchCluster { private final int numNodesPerGroup; private final ImmutableList<Group> orderedGroups; private final ImmutableMap<Integer, Group> groups; - private final ImmutableMultimap<String, Node> nodesByHost; + private final List<Node> nodes; public MockSearchCluster(String clusterId, int groups, int nodesPerGroup) { this(clusterId, createDispatchConfig(), groups, nodesPerGroup); @@ -36,21 +36,22 @@ public class MockSearchCluster extends SearchCluster { ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder(); int distributionKey = 0; + this.nodes = new ArrayList<>(); for (int group = 0; group < groups; group++) { - List<Node> nodes = new ArrayList<>(); - for (int node = 0; node < nodesPerGroup; node++) { - Node n = new Node(distributionKey, "host" + distributionKey, group); - nodes.add(n); - hostBuilder.put(n.hostname(), n); + List<Node> groupNodes = new ArrayList<>(); + for (int i = 0; i < nodesPerGroup; i++) { + Node node = new Node(distributionKey, "host" + distributionKey, group); + nodes.add(node); + groupNodes.add(node); + hostBuilder.put(node.hostname(), node); distributionKey++; } - Group g = new Group(group, nodes); + Group g = new Group(group, groupNodes); groupBuilder.put(group, g); orderedGroupBuilder.add(g); } this.orderedGroups = orderedGroupBuilder.build(); this.groups = groupBuilder.build(); - this.nodesByHost = hostBuilder.build(); this.numGroups = groups; this.numNodesPerGroup = nodesPerGroup; } @@ -61,9 +62,7 @@ public class MockSearchCluster extends SearchCluster { } @Override - public int size() { - return numGroups * numNodesPerGroup; - } + public List<Node> nodes() { return nodes; } @Override public ImmutableMap<Integer, Group> groups() { @@ -71,9 +70,7 @@ public class MockSearchCluster extends SearchCluster { } @Override - public int wantedGroupSize() { - return numNodesPerGroup; - } + public boolean allGroupsHaveSize1() { return numNodesPerGroup == 1;} @Override public int groupsWithSufficientCoverage() { diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java new file mode 100644 index 00000000000..9e5bfb4a9f5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java @@ -0,0 +1,126 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.document.GlobalId; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.searchlib.aggregation.FS4Hit; +import com.yahoo.searchlib.aggregation.Group; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.aggregation.HitsAggregationResult; +import com.yahoo.searchlib.expression.StringResultNode; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class FlatteningSearcherTestCase { + + @Test + public void testFlatteningSearcher() { + Query query = new Query("?query=test"); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping group0 = new Grouping(0); + group0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new Group().setId(new StringResultNode("unique1")) + .addAggregationResult(new HitsAggregationResult(3, "bar") + ) + ) + .addChild(new Group().setId(new StringResultNode("unique2")) + .addAggregationResult(new HitsAggregationResult(3, "bar") + ) + )); + Grouping group1 = new Grouping(0); + group1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new Group().setId(new StringResultNode("unique1")) + .addAggregationResult(new HitsAggregationResult(3, "bar") + .addHit(fs4Hit(0.7)) + .addHit(fs4Hit(0.6)) + .addHit(fs4Hit(0.3)) + ) + ) + .addChild(new Group().setId(new StringResultNode("unique2")) + .addAggregationResult(new HitsAggregationResult(3, "bar") + .addHit(fs4Hit(0.5)) + .addHit(fs4Hit(0.4)) + ) + )); + Execution execution = newExecution(new FlatteningSearcher(), + new GroupingExecutor(ComponentId.fromString("grouping")), + new ResultProvider(Arrays.asList( + new GroupingListHit(List.of(group0), null), + new GroupingListHit(List.of(group1), null)))); + Result result = execution.search(query); + assertEquals(5, result.hits().size()); + assertFlat(result); + } + + private void assertFlat(Result result) { + for (var hit : result.hits()) + assertTrue(hit instanceof FastHit); + } + + private FS4Hit fs4Hit(double relevance) { + return new FS4Hit(0, new GlobalId(new byte[GlobalId.LENGTH]), relevance); + } + + private void dump(Hit hit, String indent) { + System.out.println(indent + hit); + if (hit instanceof HitGroup) { + for (var child : (HitGroup)hit) + dump(child, indent + " "); + } + } + + private static Execution newExecution(Searcher... searchers) { + return new Execution(new SearchChain(new ComponentId("foo"), Arrays.asList(searchers)), + Execution.Context.createContextStub()); + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class ResultProvider extends Searcher { + + final Queue<GroupingListHit> hits = new LinkedList<>(); + int pass = 0; + + ResultProvider(List<GroupingListHit> hits) { + this.hits.addAll(hits); + } + + @Override + public Result search(Query query, Execution exec) { + GroupingListHit hit = hits.poll(); + for (Grouping group : hit.getGroupingList()) { + group.setFirstLevel(pass); + group.setLastLevel(pass); + } + ++pass; + Result result = exec.search(query); + result.hits().add(hit); + return result; + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java index 7f70178bcf7..1035c9d9284 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java @@ -151,7 +151,6 @@ public class HitRendererTestCase { return new Group(id, new Relevance(1)); } - @SuppressWarnings("deprecation") private static void assertRender(HitGroup hit, String expectedXml) { StringWriter str = new StringWriter(); XMLWriter out = new XMLWriter(str, 0, -1); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 637666a6a19..1d2e3c240b9 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -948,7 +948,7 @@ public class JsonRendererTestCase { } }); Group g = new Group(new StringId("Jones"), new Relevance(1.0)); - g.setField("count()", Integer.valueOf(7)); + g.setField("count()", 7); gl.add(g); rg.add(gl); r.hits().add(rg); @@ -1443,8 +1443,7 @@ public class JsonRendererTestCase { ByteArrayOutputStream bs = new ByteArrayOutputStream(); ListenableFuture<Boolean> f = renderer.render(bs, r, execution, null); assertTrue(f.get()); - String summary = Utf8.toString(bs.toByteArray()); - return summary; + return Utf8.toString(bs.toByteArray()); } @SuppressWarnings("unchecked") diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java index 1dd6eb543ef..f7876f9cddd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AccessControlService.java @@ -21,5 +21,6 @@ public interface AccessControlService { boolean requestSshAccess(TenantName tenantName); AthenzRoleInformation getAccessRoleInformation(TenantName tenantName); void setPreapprovedAccess(TenantName tenantName, boolean preapproved); + boolean getPreapprovedAccess(TenantName tenantName); Collection<AthenzUser> listMembers(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java index 415a087d990..317229f9e9a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.athenz.api.AthenzAssertion; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzGroup; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -23,14 +24,15 @@ public class AthenzAccessControlService implements AccessControlService { private static final String ALLOWED_OPERATOR_GROUPNAME = "vespa-team"; private static final String DATAPLANE_ACCESS_ROLENAME = "operator-data-plane"; private final String TENANT_DOMAIN_PREFIX = "vespa.tenant"; + private final String ACCESS_APPROVAL_POLICY = "vespa-access-requester"; private final ZmsClient zmsClient; private final AthenzRole dataPlaneAccessRole; private final AthenzGroup vespaTeam; - private final ZmsClient vespaZmsClient; //TODO: Merge ZMS clients + private final Optional<ZmsClient> vespaZmsClient; private final AthenzInstanceSynchronizer athenzInstanceSynchronizer; - public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain, ZmsClient vespaZmsClient, AthenzInstanceSynchronizer athenzInstanceSynchronizer) { + public AthenzAccessControlService(ZmsClient zmsClient, AthenzDomain domain, Optional<ZmsClient> vespaZmsClient, AthenzInstanceSynchronizer athenzInstanceSynchronizer) { this.zmsClient = zmsClient; this.vespaZmsClient = vespaZmsClient; this.athenzInstanceSynchronizer = athenzInstanceSynchronizer; @@ -66,11 +68,16 @@ public class AthenzAccessControlService implements AccessControlService { */ @Override public AthenzRoleInformation getAccessRoleInformation(TenantName tenantName) { - var role = sshRole(tenantName); - if (!vespaZmsClient.listRoles(role.domain()).contains(role)) - vespaZmsClient.createRole(role, Map.of()); + return vespaZmsClient.map( + zms -> { + var role = sshRole(tenantName); + if (!zms.listRoles(role.domain()).contains(role)) + zms.createRole(role, Map.of()); + + return zms.getFullRoleInformation(role); + } + ).orElseThrow(() -> new UnsupportedOperationException("Only allowed in systems running Vespa Athenz instance")); - return vespaZmsClient.getFullRoleInformation(role); } /** @@ -78,22 +85,25 @@ public class AthenzAccessControlService implements AccessControlService { */ @Override public boolean decideSshAccess(TenantName tenantName, Instant expiry, OAuthCredentials oAuthCredentials, boolean approve) { - var role = sshRole(tenantName); - - if (!vespaZmsClient.listRoles(role.domain()).contains(role)) - vespaZmsClient.createRole(role, Map.of()); - - if (vespaZmsClient.getMembership(role, vespaTeam)) - return false; - - var roleInformation = vespaZmsClient.getFullRoleInformation(role); - if (roleInformation.getPendingRequest().isEmpty()) - return false; - var reason = roleInformation.getPendingRequest().get().getReason(); - - vespaZmsClient.decidePendingRoleMembership(role, vespaTeam, expiry, Optional.of(reason), Optional.of(oAuthCredentials), approve); - athenzInstanceSynchronizer.synchronizeInstances(tenantName); - return true; + return vespaZmsClient.map( + zms -> { + var role = sshRole(tenantName); + if (!zms.listRoles(role.domain()).contains(role)) + zms.createRole(role, Map.of()); + + if (zms.getMembership(role, vespaTeam)) + return false; + + var roleInformation = zms.getFullRoleInformation(role); + if (roleInformation.getPendingRequest().isEmpty()) + return false; + var reason = roleInformation.getPendingRequest().get().getReason(); + + zms.decidePendingRoleMembership(role, vespaTeam, expiry, Optional.of(reason), Optional.of(oAuthCredentials), approve); + athenzInstanceSynchronizer.synchronizeInstances(tenantName); + return true; + } + ).orElseThrow(() -> new UnsupportedOperationException("Only allowed in systems running Vespa Athenz instance")); } /** @@ -101,43 +111,62 @@ public class AthenzAccessControlService implements AccessControlService { */ @Override public boolean requestSshAccess(TenantName tenantName) { - var role = sshRole(tenantName); + return vespaZmsClient.map( + zms -> { + var role = sshRole(tenantName); - if (!vespaZmsClient.listRoles(role.domain()).contains(role)) - vespaZmsClient.createRole(role, Map.of()); + if (!zms.listRoles(role.domain()).contains(role)) + zms.createRole(role, Map.of()); - if (vespaZmsClient.getMembership(role, vespaTeam)) - return false; + if (zms.getMembership(role, vespaTeam)) + return false; - vespaZmsClient.addRoleMember(role, vespaTeam, Optional.empty()); - return true; + zms.addRoleMember(role, vespaTeam, Optional.empty()); + return true; + } + ).orElseThrow(() -> new UnsupportedOperationException("Only allowed in systems running Vespa Athenz instance")); } public void setPreapprovedAccess(TenantName tenantName, boolean preapprovedAccess) { - var role = sshRole(tenantName); + vespaZmsClient.ifPresentOrElse( + zms -> { + var role = sshRole(tenantName); + var assertion = getApprovalAssertion(role); + if (preapprovedAccess) { + zms.addPolicyRule(role.domain(), ACCESS_APPROVAL_POLICY, assertion.action(), assertion.resource(), assertion.role()); + } else { + zms.deletePolicyRule(role.domain(), ACCESS_APPROVAL_POLICY, assertion.action(), assertion.resource(), assertion.role()); + } + },() -> { throw new UnsupportedOperationException("Only allowed in systems running Vespa Athenz instance"); }); + } - var attributes = Map.<String, Object>of( - "selfServe", !preapprovedAccess, - "reviewEnabled", !preapprovedAccess - ); - vespaZmsClient.createRole(role, attributes); + public boolean getPreapprovedAccess(TenantName tenantName) { + return vespaZmsClient.map( + zms -> { + var role = sshRole(tenantName); + var approvalAssertion = getApprovalAssertion(role); + return zms.getPolicy(role.domain(), ACCESS_APPROVAL_POLICY) + .map(policy -> policy.assertions().stream().anyMatch(assertion -> assertion.satisfies(approvalAssertion))) + .orElse(false); + }).orElseThrow(() -> new UnsupportedOperationException("Only allowed in systems running Vespa Athenz instance") ); } private AthenzRole sshRole(TenantName tenantName) { - return new AthenzRole(getOrCreateTenantDomain(tenantName), "ssh_access"); + return new AthenzRole(getTenantDomain(tenantName), "ssh_access"); } - private AthenzDomain getOrCreateTenantDomain(TenantName tenantName) { - var domain = new AthenzDomain(TENANT_DOMAIN_PREFIX + "." + tenantName.value()); - - if (vespaZmsClient.getDomainList(domain.getName()).isEmpty()) { - vespaZmsClient.createSubdomain(new AthenzDomain(TENANT_DOMAIN_PREFIX), tenantName.value()); - } - - return domain; + private AthenzDomain getTenantDomain(TenantName tenantName) { + return new AthenzDomain(TENANT_DOMAIN_PREFIX + "." + tenantName.value()); } public boolean isVespaTeamMember(AthenzUser user) { return zmsClient.getGroupMembership(vespaTeam, user); } + + private AthenzAssertion getApprovalAssertion(AthenzRole accessRole) { + var approverRole = new AthenzRole(accessRole.domain(), "vespa-access-approver"); + return AthenzAssertion.newBuilder(approverRole, accessRole.toResourceName(), "update_members") + .effect(AthenzAssertion.Effect.ALLOW) + .build(); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzDbMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzDbMock.java index 44271846d7d..63dfff95c03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzDbMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzDbMock.java @@ -135,6 +135,19 @@ public class AthenzDbMock { public boolean matches(AthenzIdentity principal, String action, String resource) { return assertions.stream().anyMatch(a -> a.matches(principal, action, resource)); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Policy policy = (Policy) o; + return Objects.equals(name, policy.name) && Objects.equals(assertions, policy.assertions); + } + + @Override + public int hashCode() { + return Objects.hash(name, assertions); + } } public static class Assertion { @@ -192,5 +205,18 @@ public class AthenzDbMock { public String name() { return name; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Role role = (Role) o; + return Objects.equals(name, role.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java index c14ca2bdc80..95ebe3380d4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/MockAccessControlService.java @@ -54,6 +54,11 @@ public class MockAccessControlService implements AccessControlService { } + @Override + public boolean getPreapprovedAccess(TenantName tenant) { + return false; + } + public void addPendingMember(AthenzUser user) { pendingMembers.add(user); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java index 236d5626909..541eb3dbe90 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java @@ -19,7 +19,7 @@ public class NoopRoleService implements RoleService { @Override public TenantRoles getTenantRole(TenantName tenant) { - return new TenantRoles(tenant.value() + "-legacy-host-role", tenant.value() + "-host-role", tenant.value() + "-host-service-role", tenant.value() + "-tenant-role"); + return new TenantRoles(tenant.value() + "-host-role", tenant.value() + "-host-service-role", tenant.value() + "-tenant-role"); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/TenantRoles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/TenantRoles.java index e3cd8ede07c..fe7d3862e18 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/TenantRoles.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/TenantRoles.java @@ -5,22 +5,16 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; * @author mortent */ public class TenantRoles { - private final String hostRole; private final String tenantHostRole; private final String tenantHostServiceRole; private final String containerRole; - public TenantRoles(String hostRole, String tenantHostRole, String tenantHostServiceRole, String containerRole) { - this.hostRole = hostRole; + public TenantRoles(String tenantHostRole, String tenantHostServiceRole, String containerRole) { this.tenantHostRole = tenantHostRole; this.tenantHostServiceRole = tenantHostServiceRole; this.containerRole = containerRole; } - public String hostRole() { - return hostRole; - } - public String tenantHostRole() { return tenantHostRole; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java index 2dcb80d28fb..54c4905eb46 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java @@ -23,32 +23,35 @@ public class DeploymentFailureMails { public Mail nodeAllocationFailure(RunId id, Collection<String> recipients) { return mail(id, recipients, " due to node allocation failure", "as your node resource request could not be " + - "fulfilled for your tenant. Contact Vespa Cloud support."); + "fulfilled for your tenant. Please contact Vespa Cloud support."); } public Mail deploymentFailure(RunId id, Collection<String> recipients) { return mail(id, recipients, " deployment", "and any previous deployment in the zone is unaffected. " + - "This is usually due to an invalid application configuration, or because " + - "of timeouts waiting for other deployments of the same application to finish."); + "This is usually due to an invalid application configuration. " + + "Please review warnings and errors in the deployment job log."); } public Mail installationFailure(RunId id, Collection<String> recipients) { return mail(id, recipients, "installation", - "as nodes were not able to upgrade to the new configuration. " + + "as nodes were not able to deploy to the new configuration. " + "This is often due to a misconfiguration of the components of an " + - "application, where one or more of these can not be instantiated."); + "application, where one or more of these can not be instantiated. " + + "Please check the Vespa log for errors, and contact Vespa Cloud " + + "support if unable to resolve these."); } public Mail testFailure(RunId id, Collection<String> recipients) { return mail(id, recipients, "tests", - "as one or more verification tests against the deployment failed."); + "as one or more verification tests against the deployment failed. " + + "Please review test output in the deployment job log."); } public Mail systemError(RunId id, Collection<String> recipients) { return mail(id, recipients, "due to system error", - "as something in the framework went wrong. Such errors are " + - "usually transient. Please contact the Vespa team if the problem persists!"); + "as something in the deployment framework went wrong. Such errors are " + + "usually transient. Please contact Vespa Cloud support if the problem persists."); } private Mail mail(RunId id, Collection<String> recipients, String summaryDetail, String messageDetail) { @@ -72,7 +75,8 @@ public class DeploymentFailureMails { return "System test"; if (type == JobType.stagingTest) return "Staging test"; - return "Deployment to " + type.zone(registry.system()).region(); + return (type.isDeployment() ? "Deployment to " : "Verification test of ") + + type.zone(registry.system()).region(); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 994eb1d7532..aca8425328d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -241,7 +241,8 @@ enum PathGroup { accessRequests(Matcher.tenant, "/application/v4/tenant/{tenant}/access/request/{*}"), /** Paths used to approve requests to access tenant resources */ - accessRequestApproval(Matcher.tenant, "/application/v4/tenant/{tenant}/access/approve/{*}"); + accessRequestApproval(Matcher.tenant, "/application/v4/tenant/{tenant}/access/approve/{*}", + "/application/v4/tenant/{tenant}/access/preapprove/{*}"); final List<String> pathSpecs; final List<Matcher> matchers; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 4ab7901cd62..30f416747e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -215,6 +215,7 @@ public class Application { public Optional<ApplicationVersion> oldestDeployedApplication() { return productionDeployments().values().stream().flatMap(List::stream) .map(Deployment::applicationVersion) + .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) .min(Comparator.naturalOrder()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 88df49ad3ab..5c3c43461dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -446,21 +446,17 @@ public class ApplicationController { controller.notificationsDb().removeNotifications(notification.source()); } - var oldestDeployedVersion = application.get().productionDeployments().values().stream() - .flatMap(List::stream) - .map(Deployment::applicationVersion) - .filter(version -> ! version.isDeployedDirectly()) - .min(naturalOrder()) - .orElse(ApplicationVersion.unknown); - - var olderVersions = application.get().versions().stream() - .filter(version -> version.compareTo(oldestDeployedVersion) < 0) - .sorted() - .collect(Collectors.toList()); + ApplicationVersion oldestDeployedVersion = application.get().oldestDeployedApplication() + .orElse(ApplicationVersion.unknown); + + List<ApplicationVersion> olderVersions = application.get().versions().stream() + .filter(version -> version.compareTo(oldestDeployedVersion) < 0) + .sorted() + .collect(Collectors.toList()); // Remove any version not deployed anywhere - but keep one - for (int i = 0; i < olderVersions.size() - 1; i++) { - application = application.withoutVersion(olderVersions.get(i)); + for (ApplicationVersion version : olderVersions) { + application = application.withoutVersion(version); } store(application); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 8c933f98277..be38ce0c245 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -43,7 +43,6 @@ import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static java.util.Collections.reverseOrder; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; @@ -201,7 +200,7 @@ public class DeploymentStatus { jobs.putAll(productionJobs); // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions. jobSteps.forEach((job, step) -> { - if ( ! step.isDeclared() || step.type() != StepType.test || jobs.containsKey(job)) + if ( ! step.isDeclared() || job.type().isProduction() || jobs.containsKey(job)) return; Change change = changes.get(job.application().instance()); @@ -212,9 +211,8 @@ public class DeploymentStatus { .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment()) .filter(jobId -> deploymentFor(jobId).isPresent()) .findFirst(); - Versions versions = Versions.from(change, application, firstProductionJobWithDeployment.flatMap(this::deploymentFor), systemVersion); - if (step.completedAt(change, firstProductionJobWithDeployment).isEmpty()) + if (step.completedAt(change, Optional.empty()).isEmpty()) jobs.merge(job, List.of(new Job(job.type(), versions, step.readyAt(change), change)), DeploymentStatus::union); }); return Collections.unmodifiableMap(jobs); @@ -266,16 +264,18 @@ public class DeploymentStatus { * For the "exclusive" revision upgrade policy it is the oldest such revision; otherwise, it is the latest. */ public Change outstandingChange(InstanceName instance) { - return Optional.ofNullable(instanceSteps().get(instance)) - .flatMap(instanceStatus -> application.deployableVersions(application.deploymentSpec().requireInstance(instance).revisionTarget() == next).stream() - .filter(version -> instanceStatus.dependenciesCompletedAt(Change.of(version), Optional.empty()).map(at -> ! at.isAfter(now)).orElse(false)) - .filter(version -> application.productionDeployments().getOrDefault(instance, List.of()).stream() - .noneMatch(deployment -> deployment.applicationVersion().compareTo(version) > 0)) - .map(Change::of) - .filter(change -> application.require(instance).change().application().map(change::upgrades).orElse(true)) - .filter(change -> ! hasCompleted(instance, change)) - .findFirst()) - .orElse(Change.empty()); + StepStatus status = instanceSteps().get(instance); + if (status == null) return Change.empty(); + for (ApplicationVersion version : application.deployableVersions(application.deploymentSpec().requireInstance(instance).revisionTarget() == next)) { + if (status.dependenciesCompletedAt(Change.of(version), Optional.empty()).map(now::isBefore).orElse(true)) continue; + Change change = Change.of(version); + if (application.productionDeployments().getOrDefault(instance, List.of()).stream() + .anyMatch(deployment -> change.downgrades(deployment.applicationVersion()))) continue; + if ( ! application.require(instance).change().application().map(change::upgrades).orElse(true)) continue; + if (hasCompleted(instance, change)) continue; + return change; + } + return Change.empty(); } /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ @@ -703,7 +703,7 @@ public class DeploymentStatus { return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) && (change.application().isEmpty() || change.application().equals(instance.change().application())) || step().steps().stream().noneMatch(step -> step.concerns(prod))) - ? dependenciesCompletedAt(change, dependent) + ? dependenciesCompletedAt(change, dependent).or(() -> Optional.of(Instant.EPOCH).filter(__ -> change.hasTargets())) : Optional.empty(); } @@ -805,7 +805,7 @@ public class DeploymentStatus { .orElse(false)) return job.lastCompleted().flatMap(Run::end); - return (dependent.equals(job()) ? job.lastSuccess().stream() + return (dependent.equals(job()) ? job.lastTriggered().filter(run -> run.status() == RunStatus.success).stream() : RunList.from(job).status(RunStatus.success).asList().stream()) .filter(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true) && change.application().map(run.versions().targetApplication()::equals).orElse(true)) @@ -857,10 +857,13 @@ public class DeploymentStatus { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return RunList.from(job) - .matching(run -> run.versions().targetsMatch(Versions.from(change, - status.application, - dependent.flatMap(status::deploymentFor), - status.systemVersion))) + .matching(run -> dependent.flatMap(status::deploymentFor) + .map(deployment -> run.versions().targetsMatch(Versions.from(change, + status.application, + Optional.of(deployment), + status.systemVersion))) + .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) + && (change.application().isEmpty() || change.application().get().equals(run.versions().targetApplication())))) .status(RunStatus.success) .asList().stream() .map(run -> run.end().get()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 8c7e60bd199..beef090d214 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -823,16 +823,16 @@ public class InternalStepRunner implements StepRunner { controller.notificationsDb().removeNotification(source, Notification.Type.deployment); return; case nodeAllocationFailure: - if ( ! run.id().type().environment().isTest()) updater.accept("could not allocate the requested capacity to your tenant. Contact Vespa Cloud support."); + if ( ! run.id().type().environment().isTest()) updater.accept("could not allocate the requested capacity to your tenant. Please contact Vespa Cloud support."); return; case deploymentFailed: - updater.accept("invalid application configuration, or timeout of other deployments of the same application"); + updater.accept("invalid application configuration. Please review warnings and errors in the deployment job log."); return; case installationFailed: - updater.accept("nodes were not able to upgrade to the new configuration"); + updater.accept("nodes were not able to deploy to the new configuration. Please check the Vespa log for errors, and contact Vespa Cloud support if unable to resolve these."); return; case testFailure: - updater.accept("one or more verification tests against the deployment failed"); + updater.accept("one or more verification tests against the deployment failed. Please review test output in the deployment job log."); return; case error: case endpointCertificateTimeout: @@ -840,8 +840,8 @@ public class InternalStepRunner implements StepRunner { default: logger.log(WARNING, "Don't know what to set console notification to for run status '" + run.status() + "'"); } - updater.accept("something in the framework went wrong. Such errors are " + - "usually transient. Please contact the Vespa team if the problem persists!"); + updater.accept("something in the deployment framework went wrong. Such errors are " + + "usually transient. Please contact Vespa Cloud support if the problem persists."); } private Optional<Mail> mailOf(Run run, List<String> recipients) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 66a30050f7a..647b52f7d51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -607,11 +607,7 @@ public class JobController { private void prunePackages(TenantAndApplicationId id) { controller.applications().lockApplicationIfPresent(id, application -> { - application.get().productionDeployments().values().stream() - .flatMap(List::stream) - .map(Deployment::applicationVersion) - .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) - .min(naturalOrder()) + application.get().oldestDeployedApplication() .ifPresent(oldestDeployed -> { controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); controller.applications().applicationStore().pruneTesters(id.tenant(), id.application(), oldestDeployed); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 783f34ec9ed..b996901c5d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -7,8 +7,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; @@ -49,7 +47,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final CuratorDb curator; private final SecretStore secretStore; private final EndpointCertificateProvider endpointCertificateProvider; - private final BooleanFlag deleteUnmaintainedCertificates = Flags.DELETE_UNMAINTAINED_CERTIFICATES.bindTo(controller().flagSource()); @Inject public EndpointCertificateMaintainer(Controller controller, Duration interval) { @@ -174,33 +171,22 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { matchFound = true; try (Lock lock = lock(storedApp)) { if (Optional.of(storedAppMetadata).equals(curator.readEndpointCertificateMetadata(storedApp))) { - if (deleteUnmaintainedCertificates.value()) { - log.log(Level.INFO, "Cert for app " + storedApp.serializedForm() - + " has a new leafRequestId " + unknownCertDetails.request_id() + ", updating in ZK"); - curator.writeEndpointCertificateMetadata(storedApp, storedAppMetadata.withLeafRequestId(Optional.of(unknownCertDetails.request_id()))); - } else { - log.log(Level.INFO, "Cert for app " + storedApp.serializedForm() - + " has a new leafRequestId " + unknownCertDetails.request_id()); - } + log.log(Level.INFO, "Cert for app " + storedApp.serializedForm() + + " has a new leafRequestId " + unknownCertDetails.request_id() + ", updating in ZK"); + curator.writeEndpointCertificateMetadata(storedApp, storedAppMetadata.withLeafRequestId(Optional.of(unknownCertDetails.request_id()))); } break; } } } if (!matchFound) { - if (deleteUnmaintainedCertificates.value()) { - // The certificate is not known - however it could be in the process of being requested by us or another controller. - // So we only delete if it was requested more than 7 days ago. - if (Instant.parse(providerCertificateMetadata.createTime()).isBefore(Instant.now().minus(7, ChronoUnit.DAYS))) { - log.log(Level.INFO, String.format("Deleting unmaintained certificate with request_id %s and SANs %s", - providerCertificateMetadata.requestId(), - providerCertificateMetadata.dnsNames().stream().map(d -> d.dnsName).collect(Collectors.joining(", ")))); - endpointCertificateProvider.deleteCertificate(ApplicationId.fromSerializedForm("applicationid:is:unknown"), providerCertificateMetadata.requestId()); - } - } else { - log.log(Level.INFO, () -> String.format("Found unmaintained certificate with request_id %s and SANs %s", + // The certificate is not known - however it could be in the process of being requested by us or another controller. + // So we only delete if it was requested more than 7 days ago. + if (Instant.parse(providerCertificateMetadata.createTime()).isBefore(Instant.now().minus(7, ChronoUnit.DAYS))) { + log.log(Level.INFO, String.format("Deleting unmaintained certificate with request_id %s and SANs %s", providerCertificateMetadata.requestId(), providerCertificateMetadata.dnsNames().stream().map(d -> d.dnsName).collect(Collectors.joining(", ")))); + endpointCertificateProvider.deleteCertificate(ApplicationId.fromSerializedForm("applicationid:is:unknown"), providerCertificateMetadata.requestId()); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index 7c6decb4a93..dad836ca2de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -29,11 +29,8 @@ public class TenantRoleMaintainer extends ControllerMaintainer { // Create separate athenz service for all tenants tenants.forEach(roleService::createTenantRole); - // Until we have moved to separate athenz service per tenant, make sure we update the shared policy - // to allow ssh logins for hosts in prod/perf with a separate tenant iam role. var tenantsWithRoles = tenants.stream() .map(Tenant::name) - .filter(tenant -> hasProductionDeployment(tenant) || hasPerfDeployment(tenant)) .collect(Collectors.toList()); roleService.maintainRoles(tenantsWithRoles); return 1.0; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 2ece93ba23f..c8e16634464 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -413,12 +413,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse accessRequests(String tenantName, HttpRequest request) { - if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + var tenant = TenantName.from(tenantName); + if (controller.tenants().require(tenant).type() != Tenant.Type.cloud) return ErrorResponse.badRequest("Can only see access requests for cloud tenants"); var accessControlService = controller.serviceRegistry().accessControlService(); - var accessRoleInformation = accessControlService.getAccessRoleInformation(TenantName.from(tenantName)); - var preapprovedAccess = !accessRoleInformation.isSelfServe() && !accessRoleInformation.isReviewEnabled(); + var accessRoleInformation = accessControlService.getAccessRoleInformation(tenant); + var preapprovedAccess = accessControlService.getPreapprovedAccess(tenant); var slime = new Slime(); var cursor = slime.setObject(); cursor.setBool("preapprovedAccess", preapprovedAccess); @@ -484,7 +485,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.badRequest("Can only set access privel for cloud tenants"); controller.serviceRegistry().accessControlService().setPreapprovedAccess(tenant, preapprovedAccess); - return new MessageResponse("OK"); + var slime = new Slime(); + slime.setObject().setBool("preapprovedAccess", preapprovedAccess); + return new SlimeJsonResponse(slime); } private HttpResponse tenantInfo(String tenantName, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2b52143f574..189192d8d9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -266,10 +266,12 @@ class JobControllerApiHandlerHelper { stepObject.setBool("declared", stepStatus.isDeclared()); stepObject.setString("instance", stepStatus.instance().value()); - stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); - stepStatus.readyAt(change) - .filter(controller.clock().instant()::isBefore) - .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); + // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ... + Optional<Instant> readyAt = stepStatus.job().map(jobsToRun::get).map(jobs -> jobs.get(0).readyAt()) + .orElse(stepStatus.readyAt(change)); + readyAt.ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); + readyAt.filter(controller.clock().instant()::isBefore) + .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform. @@ -304,17 +306,17 @@ class JobControllerApiHandlerHelper { for (VespaVersion available : availablePlatforms) { if ( deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber())) || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty() - || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0) + || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber())) + || change.platform().map(available.versionNumber()::compareTo).orElse(1) <= 0) break; - Cursor availableObject = availableArray.addObject(); - availableObject.setString("platform", available.versionNumber().toFullString()); + availableArray.addObject().setString("platform", available.versionNumber().toFullString()); } + change.platform().ifPresent(version -> availableArray.addObject().setString("platform", version.toFullString())); toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); } - List<ApplicationVersion> availableApplications = new ArrayList<>(application.versions()); + List<ApplicationVersion> availableApplications = new ArrayList<>(application.deployableVersions(false)); if ( ! availableApplications.isEmpty()) { - Collections.reverse(availableApplications); var latestApplication = availableApplications.get(0); Cursor latestApplicationObject = latestVersionsObject.setObject("application"); toSlime(latestApplicationObject.setObject("application"), latestApplication); @@ -326,12 +328,13 @@ class JobControllerApiHandlerHelper { for (ApplicationVersion available : availableApplications) { if ( deployments.stream().anyMatch(deployment -> deployment.applicationVersion().compareTo(available) > 0) || deployments.stream().noneMatch(deployment -> deployment.applicationVersion().compareTo(available) < 0) && ! deployments.isEmpty() - || change.application().map(available::compareTo).orElse(1) < 0) + || status.hasCompleted(stepStatus.instance(), Change.of(available)) + || change.application().map(available::compareTo).orElse(1) <= 0) break; - Cursor availableObject = availableArray.addObject(); - toSlime(availableObject.setObject("application"), available); + toSlime(availableArray.addObject().setObject("application"), available); } + change.application().ifPresent(version -> toSlime(availableArray.addObject().setObject("application"), version)); toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 21cc69369d8..989a7c31821 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -236,8 +236,13 @@ public class DeploymentContext { /** Flush count pending DNS updates */ public DeploymentContext flushDnsUpdates(int count) { var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofSeconds(count)); - dispatcher.run(); - return this; + try { + dispatcher.run(); + return this; + } + finally { + dispatcher.shutdown(); + } } /** Add a routing policy for this in given zone, with status set to inactive */ @@ -255,6 +260,11 @@ public class DeploymentContext { } /** Submit given application package for deployment */ + public DeploymentContext resubmit(ApplicationPackage applicationPackage) { + return submit(applicationPackage, Optional.of(defaultSourceRevision), salt.get()); + } + + /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { return submit(applicationPackage, Optional.of(defaultSourceRevision)); } @@ -266,7 +276,7 @@ public class DeploymentContext { /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) { - return submit(applicationPackage, sourceRevision, salt.getAndIncrement()); + return submit(applicationPackage, sourceRevision, salt.incrementAndGet()); } /** Submit given application package for deployment */ @@ -597,8 +607,9 @@ public class DeploymentContext { runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); assertFalse(jobs.run(id).get().hasFailed()); - assertEquals(job.type().isProduction(), instance().deployments().containsKey(zone)); - assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(id.application()).id())).isEmpty()); + Instance instance = tester.application(TenantAndApplicationId.from(instanceId)).require(id.application().instance()); + assertEquals(job.type().isProduction(), instance.deployments().containsKey(zone)); + assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(instance.id()).id())).isEmpty()); } private JobId jobId(JobType type) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index b2847a29654..c8126207a73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1873,7 +1873,7 @@ public class DeploymentTriggerTest { // Deploy manually again, then submit new package. app.runJob(productionCdUsEast1, cdPackage); app.submit(cdPackage); - app.runJob(systemTest); + app.triggerJobs().jobAborted(systemTest).runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); app.runJob(productionCdUsEast1) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 5be39b4a733..3b811d74f19 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -33,9 +33,6 @@ public class EndpointCertificateMaintainerTest { private final SecretStoreMock secretStore = (SecretStoreMock) tester.controller().secretStore(); private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1)); private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty()); - { - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.DELETE_UNMAINTAINED_CERTIFICATES.id(), true); - } @Test public void old_and_unused_cert_is_deleted() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 265dedec8d0..78341682f75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -945,9 +945,9 @@ public class UpgraderTest { // Application downgrades to pinned version. tester.abortAll(); - context.runJob(stagingTest).runJob(productionUsCentral1); + context.runJob(stagingTest).runJob(productionUsCentral1).runJob(productionUsWest1); assertTrue(context.instance().change().hasTargets()); - context.runJob(productionUsWest1); // us-east-3 never upgraded, so no downgrade is needed. + context.runJob(productionUsEast3); assertFalse(context.instance().change().hasTargets()); } @@ -1011,7 +1011,7 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(v2); tester.upgrader().maintain(); assertEquals(Change.of(v2), application.instance().change()); - application.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + application.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1).timeOutConvergence(productionUsWest1); tester.triggerJobs(); // While second deployment completes upgrade, version confidence becomes broken and upgrade is cancelled diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 6b13e6c951a..87fe8dd17fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -35,10 +35,12 @@ public class ControllerContainerTest { @Before public void startContainer() { - container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable); + container = JDisc.fromServicesXml(controllerServicesXml(), networking()); addUserToHostedOperatorRole(hostedOperator); } + protected Networking networking() { return Networking.disable; } + @After public void stopContainer() { container.close(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 8677b621f0d..c0a6829b026 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -579,14 +579,6 @@ }, { "application": { - "build": 3, - "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - } - }, - { - "application": { "build": 2, "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java new file mode 100644 index 00000000000..de7f2515979 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/AllowingFilter.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.playground; + +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.yolean.Exceptions; + +import java.time.Instant; +import java.util.Optional; +import java.util.Set; + +public class AllowingFilter extends JsonSecurityRequestFilterBase { + + static final AthenzPrincipal user = new AthenzPrincipal(new AthenzUser("demo")); + static final AthenzDomain domain = new AthenzDomain("domain"); + + @Override + protected Optional<ErrorResponse> filter(DiscFilterRequest request) { + try { + request.setUserPrincipal(user); + request.setAttribute(User.ATTRIBUTE_NAME, new User("mail@mail", user.getName(), "demo", null, true, -1, User.NO_DATE)); + request.setAttribute("okta.identity-token", "okta-it"); + request.setAttribute("okta.access-token", "okta-at"); + request.setAttribute(SecurityContext.ATTRIBUTE_NAME, + new SecurityContext(user, + Set.of(Role.hostedOperator()), + Instant.now().minusSeconds(3600))); + return Optional.empty(); + } + catch (Throwable t) { + return Optional.of(new ErrorResponse(500, Exceptions.toMessageString(t))); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java new file mode 100644 index 00000000000..2c91aceb8b0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/DeploymentPlayground.java @@ -0,0 +1,226 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.playground; + +import com.yahoo.application.Networking; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; + +public class DeploymentPlayground extends ControllerContainerTest { + + private final Object monitor = new Object(); + private DeploymentTester deploymentTester; + + @Override + protected Networking networking() { return Networking.enable; } + + public static void main(String[] args) throws IOException, InterruptedException { + DeploymentPlayground test = null; + try { + test = new DeploymentPlayground(); + test.startContainer(); + test.run(); + } + catch (Throwable t) { + t.printStackTrace(); + } + if (test != null && test.container != null) { + test.stopContainer(); + test.deploymentTester.runner().shutdown(); + test.deploymentTester.upgrader().shutdown(); + test.deploymentTester.readyJobsTrigger().shutdown(); + test.deploymentTester.outstandingChangeDeployer().shutdown(); + } + } + + public void run() throws IOException { + ContainerTester tester = new ContainerTester(container, ""); + deploymentTester = new DeploymentTester(new ControllerTester(tester)); + deploymentTester.controllerTester().computeVersionStatus(); + + AthenzDbMock.Domain domainMock = tester.athenzClientFactory().getSetup().getOrCreateDomain(AllowingFilter.domain); + domainMock.markAsVespaTenant(); + domainMock.admin(AllowingFilter.user.getIdentity()); + + Map<String, DeploymentContext> instances = new LinkedHashMap<>(); + for (String name : List.of("alpha", "beta", "prod5", "prod25", "prod100")) + instances.put(name, deploymentTester.newDeploymentContext("gemini", "core", name)); + + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())).deploy(); + + repl(instances); + } + + static String readDeploymentXml() throws IOException { + return Files.readString(Paths.get(System.getProperty("user.home") + "/git/" + + "vespa/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml")); + } + + void repl(Map<String, DeploymentContext> instances) throws IOException { + String[] command; + BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); + AtomicBoolean on = new AtomicBoolean(); + Thread auto = new Thread(() -> auto(instances, on)); + auto.setDaemon(true); + auto.start(); + while (true) { + try { + command = in.readLine().trim().split("\\s+"); + if (command.length == 0 || command[0].isEmpty()) continue; + synchronized (monitor) { + switch (command[0]) { + case "exit": + auto.interrupt(); + return; + case "tick": + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.upgrader().run(); + deploymentTester.triggerJobs(); + deploymentTester.runner().run(); + break; + case "run": + run(instances.get(command[1]), DeploymentContext::runJob, List.of(command).subList(2, command.length)); + break; + case "fail": + run(instances.get(command[1]), DeploymentContext::failDeployment, List.of(command).subList(2, command.length)); + break; + case "upgrade": + deploymentTester.controllerTester().upgradeSystem(new Version(command[1])); + deploymentTester.controllerTester().computeVersionStatus(); + break; + case "submit": + instances.values().iterator().next().submit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + break; + case "resubmit": + instances.values().iterator().next().resubmit(ApplicationPackageBuilder.fromDeploymentXml(readDeploymentXml())); + break; + case "advance": + deploymentTester.clock().advance(Duration.ofMinutes(Long.parseLong(command[1]))); + break; + case "auto": + switch (command[1]) { + case "on": on.set(true); break; + case "off": on.set(false); break; + default: System.err.println("Argument to 'auto' must be 'on' or 'off'"); break; + } + break; + default: + System.err.println("Cannot run '" + String.join(" ", command) + "'"); + } + } + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + + void auto(Map<String, DeploymentContext> instances, AtomicBoolean on) { + while ( ! Thread.currentThread().isInterrupted()) { + try { + synchronized (monitor) { + monitor.wait(6000); + if ( ! on.get()) + continue; + + System.err.println("auto running"); + deploymentTester.clock().advance(Duration.ofSeconds(60)); + deploymentTester.controllerTester().computeVersionStatus(); + deploymentTester.outstandingChangeDeployer().run(); + deploymentTester.upgrader().run(); + deploymentTester.triggerJobs(); + deploymentTester.runner().run(); + for (Run run : deploymentTester.jobs().active()) + if (run.versions().sourcePlatform().map(run.versions().targetPlatform()::equals).orElse(true) || Math.random() < 0.4) + instances.get(run.id().application().instance().value()).runJob(run.id().type()); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Throwable t) { + t.printStackTrace(); + } + } + } + + void run(DeploymentContext instance, BiConsumer<DeploymentContext, JobType> action, List<String> jobs) { + for (boolean triggered = true; triggered; ) { + triggered = false; + for (Run run : deploymentTester.jobs().active(instance.instanceId())) + if (jobs.isEmpty() || jobs.contains(run.id().type().jobName().replace("production-", ""))) { + action.accept(instance, run.id().type()); + triggered = true; + } + } + } + + @Override + protected String variablePartXml() { + return " <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" + + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + + " <binding>http://*/application/v4/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + + " <binding>http://*/athenz/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + + " <binding>http://*/zone/v1</binding>\n" + + " <binding>http://*/zone/v1/*</binding>\n" + + " </handler>\n" + + + " <http>\n" + + " <server id='default' port='8080' />\n" + + " <filtering>\n" + + " <request-chain id='default'>\n" + + " <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsPreflightRequestFilter'>\n" + + " <config name=\"jdisc.http.filter.security.cors.cors-filter\">" + + " <allowedUrls>\n" + + " <item>http://localhost:3000</item>\n" + + " <item>http://localhost:8080</item>\n" + + " </allowedUrls>\n" + + " </config>\n" + + " </filter>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.playground.AllowingFilter'/>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + + " <binding>http://*/*</binding>\n" + + " </request-chain>\n" + + " <response-chain id='responses'>\n" + + " <filter id='com.yahoo.jdisc.http.filter.security.cors.CorsResponseFilter'>\n" + + " <config name=\"jdisc.http.filter.security.cors.cors-filter\">" + + " <allowedUrls>\n" + + " <item>http://localhost:3000</item>\n" + + " <item>http://localhost:8080</item>\n" + + " </allowedUrls>\n" + + " </config>\n" + + " </filter>\n" + + " <binding>http://*/*</binding>\n" + + " </response-chain>\n" + + " </filtering>\n" + + " </http>\n"; + } + +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml new file mode 100644 index 00000000000..a3642acb21a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/playground/deployment.xml @@ -0,0 +1,71 @@ +<deployment> + <instance id='alpha'> + <upgrade rollout="simultaneous" revision-change="when-failing" revision-target="next" /> + <test /> + <staging /> + </instance> + <instance id='beta'> + <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="latest" /> + <prod> + <region>us-east-3</region> + <test>us-east-3</test> + </prod> + </instance> + <instance id='prod5'> + <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" /> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id='prod25'> + <upgrade rollout="simultaneous" revision-change="when-failing" revision-target="next" /> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + </parallel> + </prod> + </instance> + <instance id='prod100'> + <upgrade rollout="simultaneous" revision-change="when-clear" revision-target="next" /> + <prod> + <parallel> + <steps> + <region>us-east-3</region> + <test>us-east-3</test> + </steps> + <steps> + <region>us-central-1</region> + <test>us-central-1</test> + </steps> + <steps> + <region>us-west-1</region> + <test>us-west-1</test> + </steps> + </parallel> + </prod> + </instance> +</deployment> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 1432ea7164c..1925e5f42b6 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -86,6 +86,12 @@ function(setup_vespa_default_build_settings_fedora_36) set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_fedora_37) + message("-- Setting up default build settings for fedora 37") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) +endfunction() + function(setup_vespa_default_build_settings_amzn_2) message("-- Setting up default build settings for amzn 2") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE) @@ -206,6 +212,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_fedora_35() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 36") setup_vespa_default_build_settings_fedora_36() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 37") + setup_vespa_default_build_settings_fedora_37() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") setup_vespa_default_build_settings_amzn_2() elseif(VESPA_OS_DISTRO STREQUAL "ubuntu") diff --git a/dist/vespa.spec b/dist/vespa.spec index 74ef6a6f8fc..e6ef819b458 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -177,7 +177,14 @@ BuildRequires: gmock-devel %endif %if 0%{?fc36} BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 13.0.0 +BuildRequires: llvm-devel >= 13.0.1 +BuildRequires: boost-devel >= 1.76 +BuildRequires: gtest-devel +BuildRequires: gmock-devel +%endif +%if 0%{?fc37} +BuildRequires: protobuf-devel +BuildRequires: llvm-devel >= 13.0.1 BuildRequires: boost-devel >= 1.76 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -316,6 +323,9 @@ Requires: gtest %if 0%{?fc36} %define _vespa_llvm_version 13 %endif +%if 0%{?fc37} +%define _vespa_llvm_version 13 +%endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif @@ -434,7 +444,10 @@ Requires: llvm-libs >= 12.0.0 Requires: llvm-libs >= 13.0.0 %endif %if 0%{?fc36} -Requires: llvm-libs >= 13.0.0 +Requires: llvm-libs >= 13.0.1 +%endif +%if 0%{?fc37} +Requires: llvm-libs >= 13.0.1 %endif %endif Requires: vespa-onnxruntime = 1.7.1 diff --git a/document/abi-spec.json b/document/abi-spec.json index 83a56d4fb5e..e9bf2cb6430 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -437,6 +437,7 @@ "public java.lang.Class getValueClass()", "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", "public com.yahoo.document.StructDataType contentStruct()", + "public com.yahoo.document.StructDataType getDeclaredStructType(java.lang.String)", "public com.yahoo.document.StructDataType getHeaderType()", "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", "public boolean isA(java.lang.String)", @@ -3154,8 +3155,8 @@ "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StringFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.TensorFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ReferenceFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StructuredFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.WeightedSet)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", "public void write(com.yahoo.document.DocumentId)", diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index bb1777954a3..2fcbe7333b4 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -52,12 +52,16 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static DocumentType DOCUMENT = new DocumentType("document"); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); + final static int TAG_ID = 18; public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor // Tags are converted to weightedset<string> when reading the search definition TODO: Remove it public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); + static { + TAG.setTag(true); + } public static int lastPredefinedDataTypeId() { return 21; diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 15af1b3360b..6cfbb96ae5f 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -41,6 +41,7 @@ public class DocumentType extends StructuredDataType { private List<DocumentType> inherits = new ArrayList<>(1); private Map<String, Set<Field>> fieldSets = new HashMap<>(); private final Set<String> importedFieldNames; + private Map<String, StructDataType> declaredStructTypes = new HashMap<>(); /** * Creates a new document type and registers it with the document type manager. @@ -138,6 +139,39 @@ public class DocumentType extends StructuredDataType { return headerType; } + /** + * Get a struct declared in this document (or any inherited + * document). Returns null if no such struct was found. + * + * @param name the name of the struct + * @return reference to a struct data type, or null + **/ + public StructDataType getDeclaredStructType(String name) { + var mine = declaredStructTypes.get(name); + if (mine != null) { + return mine; + } + for (DocumentType inheritedType : inherits) { + var fromParent = inheritedType.getDeclaredStructType(name); + if (fromParent == null) { + continue; + } else if (mine == null) { + mine = fromParent; + } else if (mine != fromParent) { + throw new IllegalArgumentException("Found multiple conflicting struct types for "+name); + } + } + return mine; + } + + /** only used during configuration */ + void addDeclaredStructType(String name, StructDataType struct) { + var old = declaredStructTypes.put(name, struct); + if (old != null) { + throw new IllegalArgumentException("Already had declared struct for "+name); + } + } + /** @deprecated use contentStruct instead */ @Deprecated // TODO: Remove on Vespa 8 public StructDataType getHeaderType() { diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 9325e374daa..cb615c27116 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -542,6 +542,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DataType fieldType = getOrCreateType(fieldCfg.type()); type.addField(new Field(fieldCfg.name(), fieldCfg.internalid(), fieldType)); } + if (docType != DataType.DOCUMENT) { + docType.addDeclaredStructType(structCfg.name(), type); + } } } void fillDocument() { diff --git a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java index 35dd13efb0b..b21f059bd7d 100644 --- a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java +++ b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java @@ -24,25 +24,33 @@ public class WeightedSetDataType extends CollectionDataType { public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { this(nestedType, createIfNonExistent, removeIfZero, 0); - if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { // the tag type definition - setId(18); - } else { - setId(getName().toLowerCase().hashCode()); - } } public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero, int id) { super(createName(nestedType, createIfNonExistent, removeIfZero), id, nestedType); this.createIfNonExistent = createIfNonExistent; this.removeIfZero = removeIfZero; + if (id == 0) { + if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { // the tag type definition + setId(TAG_ID); + } else { + setId(getName().toLowerCase().hashCode()); + } + } + int code = getId(); + if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != TAG_ID)) { + throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); + } } + /* + * @deprecated // TODO remove on Vespa 8 + * Do not use - use one of the constructors above. + * Note: ignores typeName argument. + */ + @Deprecated public WeightedSetDataType(String typeName, int code, DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { - super(typeName != null ? createName(nestedType, createIfNonExistent, removeIfZero) : null, code, nestedType); - if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != 18)) // 18 == DataType.TAG.getId() is not yet initialized - throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); - this.createIfNonExistent = createIfNonExistent; - this.removeIfZero = removeIfZero; + this(nestedType, createIfNonExistent, removeIfZero, code); } @Override diff --git a/document/src/test/document/documentmanager.declstruct.cfg b/document/src/test/document/documentmanager.declstruct.cfg new file mode 100644 index 00000000000..d9c48ddc42f --- /dev/null +++ b/document/src/test/document/documentmanager.declstruct.cfg @@ -0,0 +1,89 @@ +enablecompression false +usev8geopositions false +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[1].name "common" +doctype[1].idx 10015 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "common.header" +doctype[1].structtype[1].idx 10017 +doctype[1].structtype[1].name "mystruct" +doctype[1].structtype[1].field[0].name "f0" +doctype[1].structtype[1].field[0].internalid 111558427 +doctype[1].structtype[1].field[0].type 10012 +doctype[2].name "foo" +doctype[2].idx 10018 +doctype[2].inherits[0].idx 10000 +doctype[2].inherits[1].idx 10015 +doctype[2].contentstruct 10019 +doctype[2].structtype[0].idx 10019 +doctype[2].structtype[0].name "foo.header" +doctype[2].structtype[1].idx 10020 +doctype[2].structtype[1].name "mystructinfoo" +doctype[2].structtype[1].field[0].name "f1" +doctype[2].structtype[1].field[0].internalid 1911889118 +doctype[2].structtype[1].field[0].type 10012 +doctype[3].name "bar" +doctype[3].idx 10021 +doctype[3].inherits[0].idx 10000 +doctype[3].inherits[1].idx 10015 +doctype[3].contentstruct 10022 +doctype[3].structtype[0].idx 10022 +doctype[3].structtype[0].name "bar.header" +doctype[3].structtype[1].idx 10023 +doctype[3].structtype[1].name "mystructinbar" +doctype[3].structtype[1].field[0].name "f2" +doctype[3].structtype[1].field[0].internalid 84639357 +doctype[3].structtype[1].field[0].type 10012 +doctype[4].name "foobar" +doctype[4].idx 10024 +doctype[4].inherits[0].idx 10000 +doctype[4].inherits[1].idx 10018 +doctype[4].inherits[2].idx 10021 +doctype[4].contentstruct 10025 +doctype[4].structtype[0].idx 10025 +doctype[4].structtype[0].name "foobar.header" +doctype[4].structtype[1].idx 10026 +doctype[4].structtype[1].name "mystructinfoobar" +doctype[4].structtype[1].field[0].name "f3" +doctype[4].structtype[1].field[0].internalid 63940691 +doctype[4].structtype[1].field[0].type 10012 diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index b89ed2b6b08..622c7b2237f 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -599,10 +599,50 @@ search annotationsimplicitstruct { assertTrue(array.getNestedType() instanceof StructDataType); } + @Test + public void declared_struct_types_available() { + var manager = DocumentTypeManager.fromFile("src/test/document/documentmanager.declstruct.cfg"); + var docType = manager.getDocumentType("foo"); + var struct = docType.getDeclaredStructType("mystructinfoo"); + assertNotNull(struct); + assertNotNull(struct.getField("f1")); + struct = docType.getDeclaredStructType("mystructinbar"); + assertNull(struct); + struct = docType.getDeclaredStructType("mystructinfoobar"); + assertNull(struct); + struct = docType.getDeclaredStructType("mystruct"); + assertNotNull(struct); + assertNotNull(struct.getField("f0")); + + docType = manager.getDocumentType("bar"); + struct = docType.getDeclaredStructType("mystructinfoo"); + assertNull(struct); + struct = docType.getDeclaredStructType("mystructinbar"); + assertNotNull(struct); + assertNotNull(struct.getField("f2")); + struct = docType.getDeclaredStructType("mystructinfoobar"); + assertNull(struct); + struct = docType.getDeclaredStructType("mystruct"); + assertNotNull(struct); + assertNotNull(struct.getField("f0")); + + docType = manager.getDocumentType("foobar"); + struct = docType.getDeclaredStructType("mystructinfoo"); + assertNotNull(struct); + assertNotNull(struct.getField("f1")); + struct = docType.getDeclaredStructType("mystructinbar"); + assertNotNull(struct); + assertNotNull(struct.getField("f2")); + struct = docType.getDeclaredStructType("mystructinfoobar"); + assertNotNull(struct); + assertNotNull(struct.getField("f3")); + struct = docType.getDeclaredStructType("mystruct"); + assertNotNull(struct); + assertNotNull(struct.getField("f0")); + } + // TODO test clone(). Also fieldSets not part of clone()..! // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this... - // TODO test reference to own doc type - } diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index f78fdbe89dc..f01bfb67674 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -151,5 +151,4 @@ operator>>(nbostream &is, BucketId &bucketId) } // document -template class vespalib::Array<document::BucketId>; VESPALIB_HASH_SET_INSTANTIATE_H(document::BucketId, document::BucketId::hash); diff --git a/document/src/vespa/document/bucket/bucketidlist.cpp b/document/src/vespa/document/bucket/bucketidlist.cpp index 26f2062e1c4..0f5d18f7b3a 100644 --- a/document/src/vespa/document/bucket/bucketidlist.cpp +++ b/document/src/vespa/document/bucket/bucketidlist.cpp @@ -4,9 +4,8 @@ namespace document::bucket { -BucketIdList::BucketIdList() { } BucketIdList::BucketIdList(const BucketIdList & rhs) = default; BucketIdList & BucketIdList::operator = (const BucketIdList &) = default; -BucketIdList::~BucketIdList() { } +BucketIdList::~BucketIdList() = default; } diff --git a/document/src/vespa/document/bucket/bucketidlist.h b/document/src/vespa/document/bucket/bucketidlist.h index c21879ff375..b6fdb51740a 100644 --- a/document/src/vespa/document/bucket/bucketidlist.h +++ b/document/src/vespa/document/bucket/bucketidlist.h @@ -3,17 +3,18 @@ #pragma once #include "bucketid.h" -#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/allocator.h> +#include <vector> namespace document::bucket { -using BucketIdListT = vespalib::Array<BucketId>; +using BucketIdListT = std::vector<BucketId, vespalib::allocator_large<BucketId>>; class BucketIdList : public BucketIdListT { public: - BucketIdList(); - BucketIdList(BucketIdList && rhs) = default; - BucketIdList & operator = (BucketIdList &&) = default; + using BucketIdListT::BucketIdListT; + BucketIdList(BucketIdList && rhs) noexcept = default; + BucketIdList & operator = (BucketIdList &&) noexcept = default; BucketIdList(const BucketIdList & rhs); BucketIdList & operator = (const BucketIdList &); ~BucketIdList(); diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.h b/document/src/vespa/document/fieldvalue/numericfieldvalue.h index 0a557af93d9..28981fc5286 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.h @@ -31,7 +31,7 @@ protected: public: typedef Number value_type; - NumericFieldValue(Number value=0) : NumericFieldValueBase(), _value(value), _altered(false) { } + explicit NumericFieldValue(Number value=0) : NumericFieldValueBase(), _value(value), _altered(false) { } value_type getValue() const { return _value; } void setValue(Number newValue) { _value = newValue; } diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.h b/document/src/vespa/document/fieldvalue/stringfieldvalue.h index 17a0302f6f9..07e9e578692 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.h @@ -11,7 +11,6 @@ #include <vespa/document/annotation/spantree.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/buffer.h> -#include <vespa/fastos/dynamiclibrary.h> namespace document { diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index 96e76c1e2b4..ace6c33e440 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -50,7 +50,7 @@ search book { indexing: summary | index } field year type int { - indexing: summary | index + indexing: summary | attribute } field description type string { indexing: summary | index @@ -71,10 +71,6 @@ search book { import field ref.dummy as my_dummy {} import field ref.foo as my_foo {} - field sw1 type float { - } - field didinteger type array<int> { - } rank-profile default { first-phase { expression: nativeRank @@ -87,7 +83,7 @@ search book { } field sw1 type float { - indexing: input weight * 6 + input w1 + input w2 | summary + indexing: input weight_src * 67 + input w1_src + input w2_src | summary } field didinteger type array<int> { indexing: input did | split " " | for_each { to_int } | attribute diff --git a/documentgen-test/etc/complex/common.sd b/documentgen-test/etc/complex/common.sd index a5d07c455b8..3000bd8780c 100644 --- a/documentgen-test/etc/complex/common.sd +++ b/documentgen-test/etc/complex/common.sd @@ -7,10 +7,9 @@ search common { field title type string { bolding: on indexing: index|summary - summary-to: smallsum } field mid type int { - indexing: attribute|index|summary + indexing: attribute|summary } field scorekey type string { indexing: summary @@ -43,12 +42,6 @@ search common { } annotation DocumentImplSub inherits DocumentImpl { } } - field weight type float { - } - field w1 type float { - } - field w2 type float { - } rank-profile default { first-phase { expression: nativeRank @@ -63,4 +56,8 @@ search common { field w2 type float { indexing: input w2_src + input weight_src | summary } + document-summary smallsum { + from-disk + summary title type string {} + } } diff --git a/documentgen-test/etc/complex/music2.sd b/documentgen-test/etc/complex/music2.sd index 2dc659d5a1e..34419eeea6a 100644 --- a/documentgen-test/etc/complex/music2.sd +++ b/documentgen-test/etc/complex/music2.sd @@ -31,10 +31,6 @@ search music2 { field instrument type int { } } - annotation date { - field exacttime type long { } - } - annotation place { field lat type long { } field lon type long { } diff --git a/documentgen-test/etc/complex/video.sd b/documentgen-test/etc/complex/video.sd index f49ea910128..5ca283eaa96 100644 --- a/documentgen-test/etc/complex/video.sd +++ b/documentgen-test/etc/complex/video.sd @@ -26,7 +26,7 @@ search video { indexing: index|summary } field year type int { - indexing: attribute|index|summary + indexing: attribute|summary } } diff --git a/eval/src/tests/tensor/binary_format/binary_format_test.cpp b/eval/src/tests/tensor/binary_format/binary_format_test.cpp index bec62b0e0a6..fa3b333dc72 100644 --- a/eval/src/tests/tensor/binary_format/binary_format_test.cpp +++ b/eval/src/tests/tensor/binary_format/binary_format_test.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib; diff --git a/fastos/src/tests/job.h b/fastos/src/tests/job.h index 15356270972..4546cfe1daa 100644 --- a/fastos/src/tests/job.h +++ b/fastos/src/tests/job.h @@ -28,7 +28,7 @@ public: std::mutex *mutex; std::condition_variable *condition; FastOS_ThreadInterface *otherThread, *ownThread; - int result; + std::atomic<int> result; FastOS_ThreadId _threadId; Job() diff --git a/fastos/src/tests/thread_test_base.hpp b/fastos/src/tests/thread_test_base.hpp index 49d37209b6a..eb994537f6e 100644 --- a/fastos/src/tests/thread_test_base.hpp +++ b/fastos/src/tests/thread_test_base.hpp @@ -5,7 +5,7 @@ #include <chrono> #include <thread> -static volatile int64_t number; +static std::atomic<int64_t> number; #define INCREASE_NUMBER_AMOUNT 10000 using namespace std::chrono_literals; @@ -97,11 +97,11 @@ void ThreadTestBase::Run (FastOS_ThreadInterface *thread, void *arg) guard = std::unique_lock<std::mutex>(*job->mutex); } - result = static_cast<int>(number); + result = static_cast<int>(number.load(std::memory_order_relaxed)); int sleepOn = (INCREASE_NUMBER_AMOUNT/2) * 321/10000; for (int i=0; i<(INCREASE_NUMBER_AMOUNT/2); i++) { - number = number + 2; + number.fetch_add(2, std::memory_order_relaxed); if (i == sleepOn) std::this_thread::sleep_for(1ms); diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt index 623d931e999..605b1e40a38 100644 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_library(fastos_objects OBJECT linux_file.cpp thread.cpp unix_app.cpp - unix_dynamiclibrary.cpp unix_file.cpp unix_thread.cpp ) diff --git a/fastos/src/vespa/fastos/app.cpp b/fastos/src/vespa/fastos/app.cpp index 94e467d341d..05e885a7d37 100644 --- a/fastos/src/vespa/fastos/app.cpp +++ b/fastos/src/vespa/fastos/app.cpp @@ -12,13 +12,7 @@ #include <cstring> #include <fcntl.h> -FastOS_ThreadPool *FastOS_ApplicationInterface::GetThreadPool () -{ - return _threadPool; -} - FastOS_ApplicationInterface::FastOS_ApplicationInterface() : - _threadPool(nullptr), _argc(0), _argv(nullptr) { @@ -44,13 +38,10 @@ bool FastOS_ApplicationInterface::Init () if (PreThreadInit()) { if (FastOS_Thread::InitializeClass()) { - if (FastOS_File::InitializeClass()) { - _threadPool = new FastOS_ThreadPool(128 * 1024); - rc = true; - } else - fprintf(stderr, "FastOS_File class initialization failed.\n"); - } else + rc = true; + } else { fprintf(stderr, "FastOS_Thread class initialization failed.\n"); + } } else fprintf(stderr, "FastOS_PreThreadInit failed.\n"); @@ -60,12 +51,6 @@ bool FastOS_ApplicationInterface::Init () void FastOS_ApplicationInterface::Cleanup () { - if(_threadPool != nullptr) { - _threadPool->Close(); - delete _threadPool; - _threadPool = nullptr; - } - FastOS_File::CleanupClass(); FastOS_Thread::CleanupClass(); } diff --git a/fastos/src/vespa/fastos/app.h b/fastos/src/vespa/fastos/app.h index 6aa7a9346ed..c03dbc5848c 100644 --- a/fastos/src/vespa/fastos/app.h +++ b/fastos/src/vespa/fastos/app.h @@ -10,11 +10,7 @@ #pragma once -#include <vespa/fastos/types.h> - -class FastOS_ThreadPool; - -#include <mutex> +#include "types.h" /** * FastOS application wrapper class. @@ -125,14 +121,7 @@ class FastOS_ThreadPool; class FastOS_ApplicationInterface { friend int main (int argc, char **argv); - -private: - FastOS_ApplicationInterface(const FastOS_ApplicationInterface&); - FastOS_ApplicationInterface& operator=(const FastOS_ApplicationInterface&); - protected: - FastOS_ThreadPool *_threadPool; - virtual bool PreThreadInit () { return true; } public: @@ -140,7 +129,8 @@ public: char **_argv; FastOS_ApplicationInterface(); - + FastOS_ApplicationInterface(const FastOS_ApplicationInterface&) = delete; + FastOS_ApplicationInterface& operator=(const FastOS_ApplicationInterface&) = delete; virtual ~FastOS_ApplicationInterface(); /** @@ -182,8 +172,6 @@ public: * of @ref Init() and @ref Main(). */ virtual void Cleanup (); - - FastOS_ThreadPool *GetThreadPool (); }; diff --git a/fastos/src/vespa/fastos/dynamiclibrary.h b/fastos/src/vespa/fastos/dynamiclibrary.h deleted file mode 100644 index 6762ad4ac76..00000000000 --- a/fastos/src/vespa/fastos/dynamiclibrary.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/****************************************************************-*-C++-*- - * @file - * Class definitions for FastOS_DynamicLibrary. - * - * @author Eyvind Bernhardsen - * - * Creation date : 2003-07-02 - *************************************************************************/ - - - -#pragma once - - -#include <vespa/fastos/types.h> -#include <string> - -/** - * This class contains functionality to load, get symbols from and - * unload dynamic libraries. - */ - -class FastOS_DynamicLibraryInterface -{ -public: - /** - * Destructor. The destructor will close the library if it is open. - */ - virtual ~FastOS_DynamicLibraryInterface() {} - - /** - * Open (load) the library. - * @param libname the name of the library to open - * @return Boolean success/failure - */ - virtual bool Open(const char *libname = nullptr) = 0; - - /** - * Close (unload) the library. - * @return Boolean success/failure - */ - virtual bool Close() = 0; - - /** - * Find the address of a symbol in the library. - * @param symbol Name of symbol to find - * @return Address of the symbol, or nullptr if an error has occurred - */ - virtual void * GetSymbol(const char *symbol) const = 0; - - /** - * Check if the library is open. - * @return true if it is, false if it ain't - */ - virtual bool IsOpen() const = 0; - - /** - * Return an error message describing the last error. This is - * currently platform-dependent, unfortunately; FastOS does not - * normalize the error messages. - * @return The error string if an error has occurred since the last - * invocation, or an empty one if no error has occurred. - */ - std::string GetLastErrorString(); -}; - - -# include "unix_dynamiclibrary.h" -typedef FastOS_UNIX_DynamicLibrary FASTOS_PREFIX(DynamicLibrary); - -/********************************************************************* - * Dynamic library helper macros: - * - * FASTOS_LOADABLE_EXPORT prefix that marks a symbol to be exported - * FASTOS_LOADABLE_IMPORT prefix that marks a symbol to be imported - * from a dll - * FASTOS_LOADABLE_FACTORY macro that creates and exports a function - * called factory. The macro takes a class - * name as its only parameter, and the - * factory function returns a pointer to an - * instance of that class. - * - * Example usage: - * loadableclass.h: - * class FastOS_LoadableClass - * { - * public: - * void DoSomething(); - * } - * - * in loadableclass.cpp: - * FASTOS_LOADABLE_FACTORY(LoadableClass) - *********************************************************************/ - - -# define FASTOS_LOADABLE_EXPORT - -# define FASTOS_LOADABLE_IMPORT - -#define FASTOS_LOADABLE_FACTORY(loadable_class) \ -extern "C" { \ - FASTOS_LOADABLE_EXPORT loadable_class *factory() { \ - return new loadable_class; \ - } \ -} - -// New macros to support the new gcc visibility features. - -#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) -#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp index 4f585e7daeb..ca109d831a8 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/fastos/src/vespa/fastos/file.cpp @@ -52,16 +52,6 @@ FastOS_FileInterface::FastOS_FileInterface(const char *filename) FastOS_FileInterface::~FastOS_FileInterface() = default; -bool FastOS_FileInterface::InitializeClass () -{ - return true; -} - -bool FastOS_FileInterface::CleanupClass () -{ - return true; -} - void FastOS_FileInterface::ReadBuf(void *buffer, size_t length) { diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h index 238da364d54..b5e1add3529 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/fastos/src/vespa/fastos/file.h @@ -99,20 +99,6 @@ public: void setFAdviseOptions(int options) { _fAdviseOptions = options; } /** - * Initialize the file class. This is invoked by - * @ref FastOS_Application::Init(). - * @return Boolean success/failure - */ - static bool InitializeClass (); - - /** - * Cleanup the file class. This is invoked by - * @ref FastOS_Application::Cleanup(). - * @return Boolean success/failure - */ - static bool CleanupClass (); - - /** * Copy a single file. Will overwrite destination if it already exists. * * @author Sveinar Rasmussen diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/fastos/src/vespa/fastos/linux_file.cpp index 1074c02b4ac..6b58006eb62 100644 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ b/fastos/src/vespa/fastos/linux_file.cpp @@ -413,13 +413,6 @@ FastOS_Linux_File::Open(unsigned int openFlags, const char *filename) return rc; } - -bool -FastOS_Linux_File::InitializeClass() -{ - return FastOS_UNIX_File::InitializeClass(); -} - int FastOS_Linux_File::count_open_files() { diff --git a/fastos/src/vespa/fastos/linux_file.h b/fastos/src/vespa/fastos/linux_file.h index f3c66d49f12..567af9e8ab0 100644 --- a/fastos/src/vespa/fastos/linux_file.h +++ b/fastos/src/vespa/fastos/linux_file.h @@ -40,7 +40,6 @@ public: [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; bool Open(unsigned int openFlags, const char *filename) override; - static bool InitializeClass(); static size_t getMaxDirectIOMemAlign(); static int count_open_files(); private: diff --git a/fastos/src/vespa/fastos/ringbuffer.h b/fastos/src/vespa/fastos/ringbuffer.h deleted file mode 100644 index e442faa1921..00000000000 --- a/fastos/src/vespa/fastos/ringbuffer.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -struct FastOS_RingBufferData -{ - union - { - unsigned int _messageSize; - uint8_t _buffer[1]; - }; -}; - - -class FastOS_RingBuffer -{ -private: - FastOS_RingBuffer (const FastOS_RingBuffer&); - FastOS_RingBuffer& operator=(const FastOS_RingBuffer&); - - bool _closed; - FastOS_RingBufferData *_data; - int _bufferSize; - int _dataIndex, _dataSize; - - int GetWriteIndex (int offset) - { - return (_dataIndex + _dataSize + offset) % _bufferSize; - } - - int GetReadIndex (int offset) - { - return (_dataIndex + offset) % _bufferSize; - } - - std::mutex _mutex; - -public: - void Reset () - { - _dataIndex = 0; - _dataSize = 0; - _closed = false; - } - - FastOS_RingBuffer (int bufferSize) - : _closed(false), - _data(0), - _bufferSize(bufferSize), - _dataIndex(0), - _dataSize(0), - _mutex() - { - _data = static_cast<FastOS_RingBufferData *> - (malloc(sizeof(FastOS_RingBufferData) + bufferSize)); - Reset(); - } - - ~FastOS_RingBuffer () - { - free(_data); - } - - uint8_t *GetWritePtr (int offset=0) - { - return &_data->_buffer[GetWriteIndex(offset)]; - } - - uint8_t *GetReadPtr (int offset=0) - { - return &_data->_buffer[GetReadIndex(offset)]; - } - - void Consume (int bytes) - { - _dataSize -= bytes; - _dataIndex = (_dataIndex + bytes) % _bufferSize; - } - - void Produce (int bytes) - { - _dataSize += bytes; - } - - int GetWriteSpace () - { - int spaceLeft = _bufferSize - _dataSize; - int continuousBufferLeft = _bufferSize - GetWriteIndex(0); - - if(continuousBufferLeft > spaceLeft) - continuousBufferLeft = spaceLeft; - - return continuousBufferLeft; - } - - int GetReadSpace () - { - int dataLeft = _dataSize; - int continuousBufferLeft = _bufferSize - _dataIndex; - if(continuousBufferLeft > dataLeft) - continuousBufferLeft = dataLeft; - return continuousBufferLeft; - } - - void Close () - { - _closed = true; - } - - bool GetCloseFlag () - { - return _closed; - } - - std::unique_lock<std::mutex> getGuard() { return std::unique_lock<std::mutex>(_mutex); } -}; - diff --git a/fastos/src/vespa/fastos/thread.h b/fastos/src/vespa/fastos/thread.h index 0f4cc4c09f7..f881095c29a 100644 --- a/fastos/src/vespa/fastos/thread.h +++ b/fastos/src/vespa/fastos/thread.h @@ -50,7 +50,7 @@ private: bool _closeCalledFlag; // Always lock in this order - std::mutex _freeMutex; + mutable std::mutex _freeMutex; std::mutex _liveMutex; std::condition_variable _liveCond; /** @@ -191,7 +191,10 @@ public: * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). * @return Number of currently active threads */ - int GetNumActiveThreads () const { return _numActive; } + int GetNumActiveThreads () const { + std::lock_guard<std::mutex> guard(_freeMutex); + return _numActive; + } /** * Get the number of currently inactive threads. @@ -199,7 +202,10 @@ public: * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). * @return Number of currently inactive threads */ - int GetNumInactiveThreads () const { return _numFree; } + int GetNumInactiveThreads () const { + std::lock_guard<std::mutex> guard(_freeMutex); + return _numFree; + } /** * Get the number of started threads since instantiation of the thread pool. diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h index a0806204e66..69dd3e5231c 100644 --- a/fastos/src/vespa/fastos/types.h +++ b/fastos/src/vespa/fastos/types.h @@ -4,3 +4,6 @@ #define FASTOS_PREFIX(a) FastOS_##a +// New macros to support the new gcc visibility features. +#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/fastos/src/vespa/fastos/unix_app.cpp b/fastos/src/vespa/fastos/unix_app.cpp index e94525fff22..6b1ff743506 100644 --- a/fastos/src/vespa/fastos/unix_app.cpp +++ b/fastos/src/vespa/fastos/unix_app.cpp @@ -23,9 +23,7 @@ extern char **environ; }; int -FastOS_UNIX_Application::GetOpt (const char *optionsString, - const char* &optionArgument, - int &optionIndex) +FastOS_UNIX_Application::GetOpt (const char *optionsString, const char* &optionArgument, int &optionIndex) { int rc = getopt(_argc, _argv, optionsString); optionArgument = optarg; @@ -34,16 +32,10 @@ FastOS_UNIX_Application::GetOpt (const char *optionsString, } int -FastOS_UNIX_Application::GetOptLong(const char *optionsString, - const char* &optionArgument, - int &optionIndex, - const struct option *longopts, - int *longindex) +FastOS_UNIX_Application::GetOptLong(const char *optionsString, const char* &optionArgument, int &optionIndex, + const struct option *longopts,int *longindex) { - int rc = getopt_long(_argc, _argv, optionsString, - longopts, - longindex); - + int rc = getopt_long(_argc, _argv, optionsString, longopts, longindex); optionArgument = optarg; optionIndex = optind; return rc; @@ -74,14 +66,7 @@ bool FastOS_UNIX_Application::PreThreadInit () bool FastOS_UNIX_Application::Init () { - bool rc = false; - - if(FastOS_ApplicationInterface::Init()) - { - rc = true; - } - - return rc; + return FastOS_ApplicationInterface::Init(); } void FastOS_UNIX_Application::Cleanup () diff --git a/fastos/src/vespa/fastos/unix_app.h b/fastos/src/vespa/fastos/unix_app.h index 5e076f4482b..beb494b4fea 100644 --- a/fastos/src/vespa/fastos/unix_app.h +++ b/fastos/src/vespa/fastos/unix_app.h @@ -8,9 +8,8 @@ #pragma once - -#include "types.h" #include "app.h" +#include "types.h" #include <memory> /** @@ -18,14 +17,12 @@ */ class FastOS_UNIX_Application : public FastOS_ApplicationInterface { -private: - FastOS_UNIX_Application(const FastOS_UNIX_Application&); - FastOS_UNIX_Application& operator=(const FastOS_UNIX_Application&); - protected: bool PreThreadInit () override; public: FastOS_UNIX_Application (); + FastOS_UNIX_Application(const FastOS_UNIX_Application&) = delete; + FastOS_UNIX_Application& operator=(const FastOS_UNIX_Application&) = delete; virtual ~FastOS_UNIX_Application(); /** diff --git a/fastos/src/vespa/fastos/unix_dynamiclibrary.cpp b/fastos/src/vespa/fastos/unix_dynamiclibrary.cpp deleted file mode 100644 index 68e00010211..00000000000 --- a/fastos/src/vespa/fastos/unix_dynamiclibrary.cpp +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/dynamiclibrary.h> -#include <vespa/fastos/file.h> - -#include <dlfcn.h> - -namespace { -const std::string FASTOS_DYNLIB_PREFIX("lib"); -#ifdef __APPLE__ -const std::string FASTOS_DYNLIB_SUFFIX(".dylib"); -#else -const std::string FASTOS_DYNLIB_SUFFIX(".so"); -const std::string FASTOS_DYNLIB_SUFPREFIX(".so."); -#endif - -bool hasValidSuffix(const std::string & s) -{ - if (s.rfind(FASTOS_DYNLIB_SUFFIX) == (s.size() - FASTOS_DYNLIB_SUFFIX.size())) { - return true; - } -#ifndef __APPLE__ - if (s.rfind(FASTOS_DYNLIB_SUFPREFIX) != std::string::npos) { - return true; - } -#endif - return false; -} - -} - -void -FastOS_UNIX_DynamicLibrary::SetLibName(const char *libname) -{ - if (libname != nullptr) { - _libname = libname; - if ( ! hasValidSuffix(_libname)) { - _libname.append(FASTOS_DYNLIB_SUFFIX); - } - } else { - _libname = ""; - } -} - -bool -FastOS_UNIX_DynamicLibrary::NormalizeLibName(void) -{ - bool returnCode = false; - std::string::size_type pathPos = _libname.rfind(FastOS_File::GetPathSeparator()[0]); - std::string tmp = (pathPos != std::string::npos) - ? _libname.substr(pathPos+1) - : _libname; - if (tmp.find(FASTOS_DYNLIB_PREFIX) != 0) { - tmp = FASTOS_DYNLIB_PREFIX + tmp; - if (pathPos != std::string::npos) { - tmp = _libname.substr(0, pathPos); - } - SetLibName(tmp.c_str()); - returnCode = true; - } - - return returnCode; -} - -bool -FastOS_UNIX_DynamicLibrary::Close() -{ - bool retcode = true; - - if (IsOpen()) { - retcode = (dlclose(_handle) == 0); - if (retcode) - _handle = nullptr; - } - - return retcode; -} - -FastOS_UNIX_DynamicLibrary::FastOS_UNIX_DynamicLibrary(const char *libname) : - _handle(nullptr), - _libname("") -{ - SetLibName(libname); -} - -FastOS_UNIX_DynamicLibrary::~FastOS_UNIX_DynamicLibrary() -{ - Close(); -} - -bool -FastOS_UNIX_DynamicLibrary::Open(const char *libname) -{ - if (! Close()) - return false; - if (libname != nullptr) { - SetLibName(libname); - } - - _handle = dlopen(_libname.c_str(), RTLD_NOW); - - if (_handle == nullptr) { - // Prepend "lib" if neccessary... - if (NormalizeLibName()) { - // ...try to open again if a change was made. - _handle = dlopen(_libname.c_str(), RTLD_NOW); - } - } - - return (_handle != nullptr); -} - -void * -FastOS_UNIX_DynamicLibrary::GetSymbol(const char *symbol) const -{ - return dlsym(_handle, symbol); -} - -std::string -FastOS_UNIX_DynamicLibrary::GetLastErrorString() const -{ - const char *errorString = dlerror(); - std::string e; - if (errorString != nullptr) { - e = errorString; - } - - return e; -} diff --git a/fastos/src/vespa/fastos/unix_dynamiclibrary.h b/fastos/src/vespa/fastos/unix_dynamiclibrary.h deleted file mode 100644 index 4d52a68f093..00000000000 --- a/fastos/src/vespa/fastos/unix_dynamiclibrary.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -*************************************************************-*C++-*- -* @author Eyvind Bernhardsen -* @date Creation date: 2003-07-02 -* @file -* Class definitions for FastOS_Unix_DynamicLibrary -*********************************************************************/ - - - -#pragma once - - -#include <vespa/fastos/types.h> - -class FastOS_UNIX_DynamicLibrary : public FastOS_DynamicLibraryInterface -{ -private: - FastOS_UNIX_DynamicLibrary(const FastOS_UNIX_DynamicLibrary&); - FastOS_UNIX_DynamicLibrary& operator=(const FastOS_UNIX_DynamicLibrary&); - - void *_handle; - std::string _libname; - -public: - FastOS_UNIX_DynamicLibrary(const char *libname = nullptr); - ~FastOS_UNIX_DynamicLibrary(); - - void SetLibName(const char *libname); - bool NormalizeLibName(void); - bool Close() override; - bool Open(const char *libname = nullptr) override; - void * GetSymbol(const char *symbol) const override; - std::string GetLastErrorString() const; - const char * GetLibName() const { return _libname.c_str(); } - bool IsOpen() const override { return (_handle != nullptr); } -}; - - diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4f889968897..fb75fcd8c81 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -215,13 +215,6 @@ public class Flags { TENANT_ID, CONSOLE_USER_EMAIL ); - public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( - "delete-unmaintained-certificates", true, - List.of("andreer"), "2021-09-23", "2022-03-14", - "Whether to delete certificates that are known by provider but not by controller", - "Takes effect on next run of EndpointCertificateMaintainer" - ); - public static final UnboundBooleanFlag UNORDERED_MERGE_CHAINING = defineFeatureFlag( "unordered-merge-chaining", true, List.of("vekterli", "geirst"), "2021-11-15", "2022-06-01", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 3ae85a54453..225c45fbfdb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -240,6 +240,13 @@ public class PermanentFlags { "Takes effect immediately.", ZONE_ID, APPLICATION_ID); + public static final UnboundListFlag<Integer> INCOMPATIBLE_MAJOR_VERSIONS = defineListFlag( + "incompatible-major-versions", List.of(8), Integer.class, + "A list of major versions which are binary-incompatible and requires an application package to " + + "be built specifically for that Vespa version. When an application upgrades to an incompatible major " + + "version, the config server will refuse to serve config to nodes still running on older major versions", + "Takes effect immediately"); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h index d582f7cfbcb..a095c274687 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.h +++ b/fnet/src/vespa/fnet/frt/rpcrequest.h @@ -8,7 +8,6 @@ #include <vespa/vespalib/util/stash.h> #include <atomic> -class FNETConnection; class FNET_Packet; class FRT_IAbortHandler diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/DeactivatedContainer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/DeactivatedContainer.java index f85f626d4b7..fa749d924d1 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/DeactivatedContainer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/DeactivatedContainer.java @@ -29,7 +29,7 @@ public interface DeactivatedContainer { * DeactivatedContainer is considered to have terminated once there are no more {@link Request}s, {@link Response}s * or corresponding {@link ContentChannel}s being processed by components that belong to it.</p> * - * <p>If termination has already occured, this method immediately runs the given Runnable in the current thread.</p> + * <p>If termination has already occurred, this method immediately runs the given Runnable in the current thread.</p> * * @param task The task to run once this DeactivatedContainer has terminated. */ diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index 0853413ec68..384abdc31e9 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -XX:-OmitStackTraceInFastThrow -Djava.io.tmpdir=${VESPA_HOME}/tmp" +addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=448 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -XX:-OmitStackTraceInFastThrow -Djava.io.tmpdir=${VESPA_HOME}/tmp" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/messagebus/src/vespa/messagebus/sourcesession.cpp b/messagebus/src/vespa/messagebus/sourcesession.cpp index 0cf7135a1db..0691e0c07f9 100644 --- a/messagebus/src/vespa/messagebus/sourcesession.cpp +++ b/messagebus/src/vespa/messagebus/sourcesession.cpp @@ -9,7 +9,6 @@ using vespalib::make_string; - namespace mbus { SourceSession::SourceSession(MessageBus &mbus, const SourceSessionParams ¶ms) @@ -75,26 +74,29 @@ SourceSession::send(Message::UP msg) if (msg->getTimeRemaining() == 0ms) { msg->setTimeRemaining(_timeout); } + uint32_t my_pending_count = 0; { std::lock_guard guard(_lock); if (_closed) { return Result(Error(ErrorCode::SEND_QUEUE_CLOSED, "Source session is closed."), std::move(msg)); } - if (_throttlePolicy && !_throttlePolicy->canSend(*msg, _pendingCount)) { + my_pending_count = getPendingCount(); + if (_throttlePolicy && !_throttlePolicy->canSend(*msg, my_pending_count)) { return Result(Error(ErrorCode::SEND_QUEUE_FULL, - make_string("Too much pending data (%d messages).", _pendingCount)), + make_string("Too much pending data (%d messages).", my_pending_count)), std::move(msg)); } msg->pushHandler(_replyHandler); if (_throttlePolicy) { _throttlePolicy->processMessage(*msg); } - ++_pendingCount; + ++my_pending_count; + _pendingCount.store(my_pending_count, std::memory_order_relaxed); } if (msg->getTrace().shouldTrace(TraceLevel::COMPONENT)) { msg->getTrace().trace(TraceLevel::COMPONENT, make_string("Source session accepted a %d byte message. %d message(s) now pending.", - msg->getApproxSize(), _pendingCount)); + msg->getApproxSize(), my_pending_count)); } msg->pushHandler(*this); _sequencer.handleMessage(std::move(msg)); @@ -105,25 +107,28 @@ void SourceSession::handleReply(Reply::UP reply) { bool done; + uint32_t my_pending_count = 0; { std::lock_guard guard(_lock); - assert(_pendingCount > 0); - --_pendingCount; + my_pending_count = getPendingCount(); + assert(my_pending_count > 0); + --my_pending_count; + _pendingCount.store(my_pending_count, std::memory_order_relaxed); if (_throttlePolicy) { _throttlePolicy->processReply(*reply); } - done = (_closed && _pendingCount == 0); + done = (_closed && my_pending_count == 0); } if (reply->getTrace().shouldTrace(TraceLevel::COMPONENT)) { reply->getTrace().trace(TraceLevel::COMPONENT, - make_string("Source session received reply. %d message(s) now pending.", _pendingCount)); + make_string("Source session received reply. %d message(s) now pending.", my_pending_count)); } IReplyHandler &handler = reply->getCallStack().pop(*reply); handler.handleReply(std::move(reply)); if (done) { { std::lock_guard guard(_lock); - assert(_pendingCount == 0); + assert(getPendingCount() == 0); assert(_closed); _done = true; } @@ -136,7 +141,7 @@ SourceSession::close() { std::unique_lock guard(_lock); _closed = true; - if (_pendingCount == 0) { + if (getPendingCount() == 0) { _done = true; } while (!_done) { diff --git a/messagebus/src/vespa/messagebus/sourcesession.h b/messagebus/src/vespa/messagebus/sourcesession.h index f75f41e2d20..364533ece17 100644 --- a/messagebus/src/vespa/messagebus/sourcesession.h +++ b/messagebus/src/vespa/messagebus/sourcesession.h @@ -5,6 +5,7 @@ #include "result.h" #include "sequencer.h" #include "sourcesessionparams.h" +#include <atomic> #include <condition_variable> namespace mbus { @@ -23,15 +24,15 @@ private: std::mutex _lock; std::condition_variable _cond; - MessageBus &_mbus; - ReplyGate *_gate; - Sequencer _sequencer; - IReplyHandler &_replyHandler; - IThrottlePolicy::SP _throttlePolicy; - duration _timeout; - uint32_t _pendingCount; - bool _closed; - bool _done; + MessageBus &_mbus; + ReplyGate *_gate; + Sequencer _sequencer; + IReplyHandler &_replyHandler; + IThrottlePolicy::SP _throttlePolicy; + duration _timeout; + std::atomic<uint32_t> _pendingCount; + bool _closed; + bool _done; private: /** @@ -113,7 +114,9 @@ public: * * @return The pending count. */ - uint32_t getPendingCount() const { return _pendingCount; } + [[nodiscard]] uint32_t getPendingCount() const noexcept { + return _pendingCount.load(std::memory_order_relaxed); + } /** * Sets the number of seconds a message can be attempted sent until it times out. diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostParser.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostParser.java index c149683d5e6..caa7129ad34 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostParser.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/xgboost/XGBoostParser.java @@ -15,12 +15,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; */ class XGBoostParser { - private List<XGBoostTree> xgboostTrees; + private final List<XGBoostTree> xgboostTrees; /** * Constructor stores parsed JSON trees. * - * @param filePath XGBoost JSON output file. + * @param filePath XGBoost JSON intput file. * @throws JsonProcessingException Fails JSON parsing. * @throws IOException Fails file reading. */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java index ef294d8bd2e..b76c48c3839 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java @@ -7,7 +7,7 @@ import java.time.Instant; import java.util.Objects; /** - * This class should be used by exactly 2 thread, 1 for each interface it implements. + * This class should be used by exactly 2 threads, 1 for each interface it implements. * * @author freva */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 9b988e9a379..ada36af89e1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -142,8 +142,7 @@ public class NodeAgentImpl implements NodeAgent { loopThread = new Thread(() -> { while (!terminated.get()) { try { - NodeAgentContext context = contextSupplier.nextContext(); - converge(context); + converge(contextSupplier.nextContext()); } catch (InterruptedException ignored) { } } }); @@ -271,7 +270,7 @@ public class NodeAgentImpl implements NodeAgent { } String output = containerOperations.restartVespa(context); - if (!output.isBlank()) { + if ( ! output.isBlank()) { context.log(logger, "Restart services output: " + output); } currentRestartGeneration = context.node().wantedRestartGeneration(); @@ -282,7 +281,7 @@ public class NodeAgentImpl implements NodeAgent { return existingContainer; } - private Optional<String> shouldRestartServices( NodeAgentContext context, Container existingContainer) { + private Optional<String> shouldRestartServices(NodeAgentContext context, Container existingContainer) { NodeSpec node = context.node(); if (!existingContainer.state().isRunning() || node.state() != NodeState.active) return Optional.empty(); @@ -578,18 +577,6 @@ public class NodeAgentImpl implements NodeAgent { return temp; } - // TODO: Also skip orchestration if we're downgrading in test/staging - // How to implement: - // - test/staging: We need to figure out whether we're in test/staging, zone is available in Environment - // - downgrading: Impossible to know unless we look at the hosted version, which is - // not available in the docker image (nor its name). Not sure how to solve this. Should - // the node repo return the hosted version or a downgrade bit in addition to - // wanted docker image etc? - // Should the tenant pipeline instead use BCP tool to upgrade faster!? - // - // More generally, the node repo response should contain sufficient info on what the docker image is, - // to allow the node admin to make decisions that depend on the docker image. Or, each docker image - // needs to contain routines for drain and suspend. For many images, these can just be dummy routines. private void orchestratorSuspendNode(NodeAgentContext context) { if (context.node().state() != NodeState.active) return; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 04b03afa7aa..fd93252f3b5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -33,13 +33,15 @@ import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Logger; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.when; /** @@ -49,10 +51,10 @@ import static org.mockito.Mockito.when; public class ContainerTester implements AutoCloseable { private static final Logger log = Logger.getLogger(ContainerTester.class.getName()); - private static final Duration INTERVAL = Duration.ofMillis(10); static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com"); private final Thread loopThread; + private final Phaser phaser = new Phaser(1); private final ContainerEngineMock containerEngine = new ContainerEngineMock(); private final FileSystem fileSystem = TestFileSystem.create(); @@ -66,7 +68,6 @@ public class ContainerTester implements AutoCloseable { final NodeAdminStateUpdater nodeAdminStateUpdater; final NodeAdminImpl nodeAdmin; - private boolean terminated = false; private volatile NodeAdminStateUpdater.State wantedState = NodeAdminStateUpdater.State.RESUMED; @@ -86,11 +87,20 @@ public class ContainerTester implements AutoCloseable { Metrics metrics = new Metrics(); FileSystem fileSystem = TestFileSystem.create(); - NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl( - contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, - storageMaintainer, flagSource, - Collections.emptyList(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1), - VespaServiceDumper.DUMMY_INSTANCE); + NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> + new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, + storageMaintainer, flagSource, + Collections.emptyList(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1), + VespaServiceDumper.DUMMY_INSTANCE) { + @Override public void converge(NodeAgentContext context) { + super.converge(context); + phaser.arriveAndAwaitAdvance(); + } + @Override public void stopForHostSuspension(NodeAgentContext context) { + super.stopForHostSuspension(context); + phaser.arriveAndDeregister(); + } + }; nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); @@ -99,19 +109,14 @@ public class ContainerTester implements AutoCloseable { loopThread = new Thread(() -> { nodeAdminStateUpdater.start(); - - while (! terminated) { + while ( !phaser.isTerminated()) { try { nodeAdminStateUpdater.converge(wantedState); } catch (RuntimeException e) { log.info(e.getMessage()); } - try { - Thread.sleep(INTERVAL.toMillis()); - } catch (InterruptedException e) { - e.printStackTrace(); - } } + nodeAdminStateUpdater.stop(); }); loopThread.start(); } @@ -124,6 +129,10 @@ public class ContainerTester implements AutoCloseable { ", but that image does not exist in the container engine"); } } + + if (nodeRepository.getOptionalNode(nodeSpec.hostname()).isEmpty()) + phaser.register(); + nodeRepository.updateNodeSpec(new NodeSpec.Builder(nodeSpec) .parentHostname(HOST_HOSTNAME.value()) .build()); @@ -134,10 +143,19 @@ public class ContainerTester implements AutoCloseable { } <T> T inOrder(T t) { - // Should probably wait for one NodeAdminStateUpdater tick & one node agent tick - // instead of waiting up to a timeout, but there are currently no guarantees on the node - // agent ticks. - return inOrder.verify(t, timeout(60000)); + waitSomeTicks(); + return inOrder.verify(t); + } + + void waitSomeTicks() { + try { + // 3 is enough for everyone! (Well, maybe not for all eternity ...) + for (int i = 0; i < 3; i++) + phaser.awaitAdvanceInterruptibly(phaser.arrive(), 1000, TimeUnit.MILLISECONDS); + } + catch (InterruptedException | TimeoutException e) { + throw new RuntimeException(e); + } } public static NodeAgentContext containerMatcher(ContainerName containerName) { @@ -146,17 +164,12 @@ public class ContainerTester implements AutoCloseable { @Override public void close() { - // First, stop NodeAdmin and all the NodeAgents - nodeAdminStateUpdater.stop(); - - terminated = true; - do { - try { - loopThread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } while (loopThread.isAlive()); + phaser.forceTermination(); + try { + loopThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java index de769d6a2d0..5bb279323e9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java @@ -25,8 +25,7 @@ public class MultiContainerTest { DockerImage image2 = DockerImage.fromString("registry.example.com/image2"); try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) { addAndWaitForNode(tester, "host1.test.yahoo.com", image1); - NodeSpec nodeSpec2 = addAndWaitForNode( - tester, "host2.test.yahoo.com", image2); + NodeSpec nodeSpec2 = addAndWaitForNode(tester, "host2.test.yahoo.com", image2); tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java index aec9336c13c..4d95991dd29 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java @@ -33,9 +33,7 @@ public class RebootTest { ContainerName host1 = new ContainerName("host1"); tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any(), any()); - try { - tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); - } catch (RuntimeException ignored) { } + tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); tester.inOrder(tester.orchestrator).suspend( eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index d87a60e4a44..28ef19d4065 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -784,8 +784,8 @@ public class NodeAgentImplTest { return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, storageMaintainer, flagSource, - List.of(credentialsMaintainer), Optional.of(aclMaintainer), - Optional.of(healthChecker), clock, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE); + List.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker), + clock, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index ba8aa49d87c..338187c5270 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -122,6 +123,10 @@ public class NodeRepoStats { public static class ApplicationStats implements Comparable<ApplicationStats> { + private static final Comparator<ApplicationStats> comparator = Comparator.comparingDouble(ApplicationStats::unutilizedCost).reversed() + .thenComparingDouble(ApplicationStats::cost) + .thenComparing(ApplicationStats::id); + private final ApplicationId id; private final Load load; private final double cost; @@ -148,7 +153,7 @@ public class NodeRepoStats { @Override public int compareTo(NodeRepoStats.ApplicationStats other) { - return -Double.compare(this.unutilizedCost(), other.unutilizedCost()); + return comparator.compare(this, other); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index dc39aa722c7..ce6ac9d5f5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -23,12 +23,10 @@ public interface HostProvisioner { /** The host must be provisioned exclusively for the applicationId */ exclusive, - /** The host must be provisioned to be shared with other applications. \ - */ + /** The host must be provisioned to be shared with other applications. */ shared, - /** The client has no requirements on whether the host must be provisio\ - ned exclusively or shared. */ + /** The client has no requirements on whether the host must be provisioned exclusively or shared. */ any } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 7643368aad9..b5fd8c8111f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -73,27 +73,17 @@ public class ProvisionedHost { return Node.reserve(Set.of(), nodeHostname(), hostHostname, nodeResources, hostType.childNodeType()).build(); } - public String getId() { - return id; - } - - public String hostHostname() { - return hostHostname; - } - - public Flavor hostFlavor() { - return hostFlavor; - } - - public String nodeHostname() { - return nodeAddresses.get(0).hostname(); - } - - public List<Address> nodeAddresses() { - return nodeAddresses; - } - + public String getId() { return id; } + public String hostHostname() { return hostHostname; } + public Flavor hostFlavor() { return hostFlavor; } + public NodeType hostType() { return hostType; } + public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } + public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } + public List<Address> nodeAddresses() { return nodeAddresses; } public NodeResources nodeResources() { return nodeResources; } + public Version osVersion() { return osVersion; } + + public String nodeHostname() { return nodeAddresses.get(0).hostname(); } @Override public boolean equals(Object o) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json index 8867520fef6..8a46f8115be 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json @@ -11,7 +11,7 @@ }, "applications": [ { - "id": "tenant3.application3.instance3", + "id": "tenant1.application1.instance1", "load": { "cpu": 0.0, "memory": 0.0, @@ -31,7 +31,7 @@ "unutilizedCost": 0.0 }, { - "id": "tenant4.application4.instance4", + "id": "tenant3.application3.instance3", "load": { "cpu": 0.0, "memory": 0.0, @@ -41,7 +41,7 @@ "unutilizedCost": 0.0 }, { - "id": "tenant1.application1.instance1", + "id": "tenant4.application4.instance4", "load": { "cpu": 0.0, "memory": 0.0, diff --git a/parent/pom.xml b/parent/pom.xml index cac01fc6d5e..c16f812f80c 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -75,9 +75,6 @@ <artifactId>maven-compiler-plugin</artifactId> <version>${maven-compiler-plugin.version}</version> <configuration> - <jdkToolchain> - <version>11</version> - </jdkToolchain> <release>11</release> <showWarnings>true</showWarnings> <optimize>true</optimize> diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index d947ca51f49..33770004aba 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -318,14 +318,14 @@ DummyPersistence::listBuckets(BucketSpace bucketSpace) const list.push_back(entry.first); } } - return BucketIdListResult(list); + return BucketIdListResult(std::move(list)); } void -DummyPersistence::setModifiedBuckets(const BucketIdListResult::List& buckets) +DummyPersistence::setModifiedBuckets(BucketIdListResult::List buckets) { std::lock_guard lock(_monitor); - _modifiedBuckets = buckets; + _modifiedBuckets = std::move(buckets); } void DummyPersistence::set_fake_bucket_set(const std::vector<std::pair<Bucket, BucketInfo>>& fake_info) { @@ -348,10 +348,10 @@ DummyPersistence::getModifiedBuckets(BucketSpace bucketSpace) const { std::lock_guard lock(_monitor); if (bucketSpace == FixedBucketSpaces::default_space()) { - return BucketIdListResult(_modifiedBuckets); + return BucketIdListResult(std::move(_modifiedBuckets)); } else { BucketIdListResult::List emptyList; - return BucketIdListResult(emptyList); + return BucketIdListResult(BucketIdListResult::List()); } } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 3b6fc9f1449..683a738255f 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -138,12 +138,12 @@ class DummyPersistence : public AbstractPersistenceProvider { public: DummyPersistence(const std::shared_ptr<const document::DocumentTypeRepo>& repo); - ~DummyPersistence(); + ~DummyPersistence() override; Result initialize() override; BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; - void setModifiedBuckets(const BucketIdListResult::List& result); + void setModifiedBuckets(BucketIdListResult::List result); // Important: any subsequent mutations to the bucket set in fake_info will reset // the bucket info due to implicit recalculation of bucket info. diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 35654240ec7..6ee99e49798 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -18,8 +18,7 @@ AbstractPersistenceProvider::removeIfFoundAsync(const Bucket& b, Timestamp times BucketIdListResult AbstractPersistenceProvider::getModifiedBuckets(BucketSpace) const { - BucketIdListResult::List list; - return BucketIdListResult(list); + return BucketIdListResult(BucketIdListResult::List()); } } diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index 10c589307ba..5046a380c86 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -71,7 +71,7 @@ std::ostream & operator << (std::ostream & os, const Result & r); std::ostream & operator << (std::ostream & os, const Result::ErrorType &errorCode); -class BucketInfoResult : public Result { +class BucketInfoResult final : public Result { public: /** * Constructor to use for a result where an error has been detected. @@ -95,7 +95,7 @@ private: BucketInfo _info; }; -class UpdateResult : public Result +class UpdateResult final : public Result { public: /** @@ -151,7 +151,7 @@ private: uint32_t _numRemoved; }; -class GetResult : public Result { +class GetResult final : public Result { public: /** * Constructor to use when there was an error retrieving the document. @@ -173,6 +173,8 @@ public: _is_tombstone(false) { } + GetResult(GetResult &&) noexcept = default; + GetResult & operator=(GetResult &&) noexcept = default; /** * Constructor to use when we found the document asked for. @@ -225,7 +227,7 @@ private: bool _is_tombstone; }; -class BucketIdListResult : public Result { +class BucketIdListResult final : public Result { public: using List = document::bucket::BucketIdList; @@ -241,12 +243,16 @@ public: * @param list The list of bucket ids this partition has. Is swapped with * the list internal to this object. */ - BucketIdListResult(List& list) - : Result() - { - _info.swap(list); - } - + BucketIdListResult(List list) + : Result(), + _info(std::move(list)) + { } + BucketIdListResult() + : Result(), + _info() + { } + BucketIdListResult(BucketIdListResult &&) noexcept = default; + BucketIdListResult & operator =(BucketIdListResult &&) noexcept = default; ~BucketIdListResult(); const List& getList() const { return _info; } @@ -278,7 +284,7 @@ private: IteratorId _iterator; }; -class IterateResult : public Result { +class IterateResult final : public Result { public: using List = std::vector<std::unique_ptr<DocEntry>>; diff --git a/screwdriver.yaml b/screwdriver.yaml index 57b7a3c05ae..428b1423603 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -91,7 +91,7 @@ jobs: screwdriver.cd/timeout: 90 screwdriver.cd/dockerEnabled: true screwdriver.cd/dockerCpu: TURBO - screwdriver.cd/dockerRam: TURBO + screwdriver.cd/dockerRam: HIGH environment: LOCAL_MVN_REPO: "/tmp/vespa/mvnrepo" VESPA_MAVEN_EXTRA_OPTS: "--show-version --batch-mode --no-snapshot-updates -Dmaven.repo.local=/tmp/vespa/mvnrepo" @@ -190,7 +190,7 @@ jobs: screwdriver.cd/timeout: 300 screwdriver.cd/dockerEnabled: true screwdriver.cd/dockerCpu: TURBO - screwdriver.cd/dockerRam: TURBO + screwdriver.cd/dockerRam: HIGH secrets: - COPR_WEBHOOK diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index c5c50ff596d..5c20c302b73 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -13,6 +13,7 @@ #include <cassert> #include <getopt.h> #include <vector> +#include <limits> #include <vespa/log/log.h> LOG_SETUP("vespa-gen-testdocs"); diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp index 6f6fe3d4e41..c2707a9eecb 100644 --- a/searchcore/src/tests/proton/index/indexcollection_test.cpp +++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/matching/fakesearchcontext.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> +#include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchcorespi/index/warmupindexcollection.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/size_literals.h> @@ -15,6 +17,9 @@ using search::FixedSourceSelector; using search::index::FieldLengthInfo; using search::queryeval::FakeSearchable; using search::queryeval::ISourceSelector; +using search::queryeval::FakeRequestContext; +using search::queryeval::FieldSpecList; +using search::queryeval::FieldSpec; using searchcorespi::index::WarmupConfig; class MockIndexSearchable : public FakeIndexSearchable { @@ -46,38 +51,41 @@ public: vespalib::TestClock _clock; std::shared_ptr<IndexSearchable> _warmup; - void expect_searchable_can_be_appended(IndexCollection::UP collection) { + void expect_searchable_can_be_appended(ISearchableIndexCollection & collection) { const uint32_t id = 42; - collection->append(id, _source1); - EXPECT_EQ(1u, collection->getSourceCount()); - EXPECT_EQ(id, collection->getSourceId(0)); + collection.append(id, _source1); + EXPECT_EQ(1u, collection.getSourceCount()); + EXPECT_EQ(id, collection.getSourceId(0)); } - void expect_searchable_can_be_replaced(IndexCollection::UP collection) { + void expect_searchable_can_be_replaced(ISearchableIndexCollection & collection) { const uint32_t id = 42; - collection->append(id, _source1); - EXPECT_EQ(1u, collection->getSourceCount()); - EXPECT_EQ(id, collection->getSourceId(0)); - EXPECT_EQ(_source1.get(), &collection->getSearchable(0)); + collection.append(id, _source1); + EXPECT_EQ(1u, collection.getSourceCount()); + EXPECT_EQ(id, collection.getSourceId(0)); + EXPECT_EQ(_source1.get(), &collection.getSearchable(0)); - collection->replace(id, _source2); - EXPECT_EQ(1u, collection->getSourceCount()); - EXPECT_EQ(id, collection->getSourceId(0)); - EXPECT_EQ(_source2.get(), &collection->getSearchable(0)); + collection.replace(id, _source2); + EXPECT_EQ(1u, collection.getSourceCount()); + EXPECT_EQ(id, collection.getSourceId(0)); + EXPECT_EQ(_source2.get(), &collection.getSearchable(0)); } - IndexCollection::UP make_unique_collection() const { + std::unique_ptr<IndexCollection> + make_unique_collection() const { return std::make_unique<IndexCollection>(_selector); } - IndexCollection::SP make_shared_collection() const { + std::shared_ptr<IndexCollection> + make_shared_collection() const { return std::make_shared<IndexCollection>(_selector); } - IndexCollection::UP create_warmup(const IndexCollection::SP& prev, const IndexCollection::SP& next) { - return std::make_unique<WarmupIndexCollection>(WarmupConfig(1s, false), prev, next, *_warmup, _executor, _clock.clock(), *this); + std::shared_ptr<WarmupIndexCollection> + create_warmup(const IndexCollection::SP& prev, const IndexCollection::SP& next) { + return std::make_shared<WarmupIndexCollection>(WarmupConfig(1s, false), prev, next, *_warmup, _executor, _clock.clock(), *this); } void warmupDone(std::shared_ptr<WarmupIndexCollection> current) override { @@ -98,19 +106,19 @@ public: TEST_F(IndexCollectionTest, searchable_can_be_appended_to_normal_collection) { - expect_searchable_can_be_appended(make_unique_collection()); + expect_searchable_can_be_appended(*make_unique_collection()); } TEST_F(IndexCollectionTest, searchable_can_be_replaced_in_normal_collection) { - expect_searchable_can_be_replaced(make_unique_collection()); + expect_searchable_can_be_replaced(*make_unique_collection()); } TEST_F(IndexCollectionTest, searchable_can_be_appended_to_warmup_collection) { auto prev = make_shared_collection(); auto next = make_shared_collection(); - expect_searchable_can_be_appended(create_warmup(prev, next)); + expect_searchable_can_be_appended(*create_warmup(prev, next)); EXPECT_EQ(0u, prev->getSourceCount()); EXPECT_EQ(1u, next->getSourceCount()); } @@ -119,7 +127,7 @@ TEST_F(IndexCollectionTest, searchable_can_be_replaced_in_warmup_collection) { auto prev = make_shared_collection(); auto next = make_shared_collection(); - expect_searchable_can_be_replaced(create_warmup(prev, next)); + expect_searchable_can_be_replaced(*create_warmup(prev, next)); EXPECT_EQ(0u, prev->getSourceCount()); EXPECT_EQ(1u, next->getSourceCount()); } @@ -162,4 +170,20 @@ TEST_F(IndexCollectionTest, returns_empty_field_length_info_when_no_searchables_ EXPECT_EQ(0, collection->get_field_length_info("foo").get_num_samples()); } +TEST_F(IndexCollectionTest, warmup_can_create_blueprint) +{ + auto prev = make_shared_collection(); + auto next = make_shared_collection(); + auto indexcollection = create_warmup(prev, next); + const uint32_t id = 42; + indexcollection->append(id, _source1); + + FakeRequestContext requestContext; + FieldSpecList fields; + fields.add(FieldSpec("dummy", 1, search::fef::IllegalHandle)); + search::query::SimpleStringTerm term("what", "dummy", 1, search::query::Weight(100)); + auto blueprint = indexcollection->createBlueprint(requestContext, fields, term); + EXPECT_TRUE(blueprint); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index e8cc1b54235..ff65de90c4a 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -30,7 +30,7 @@ struct DummyPersistenceHandler : public IPersistenceHandler { RetrieversSP getDocumentRetrievers(storage::spi::ReadConsistency) override { return RetrieversSP(); } void handleListActiveBuckets(IBucketIdListResultHandler &) override {} - void handlePopulateActiveBuckets(document::BucketId::List &, IGenericResultHandler &) override {} + void handlePopulateActiveBuckets(document::BucketId::List, IGenericResultHandler &) override {} }; BucketSpace space_1(1); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index ce0b09a48b8..95593307ba2 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -217,7 +217,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { } void handleListBuckets(IBucketIdListResultHandler &resultHandler) override { - resultHandler.handle(BucketIdListResult(bucketList)); + resultHandler.handle(BucketIdListResult(BucketId::List(bucketList.begin(), bucketList.end()))); } void handleSetClusterState(const ClusterState &calc, IGenericResultHandler &resultHandler) override { @@ -245,7 +245,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { } void handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler) override { - resultHandler.handle(BucketIdListResult(modBucketList)); + resultHandler.handle(BucketIdListResult(std::move(modBucketList))); } void handleSplit(FeedToken token, const storage::spi::Bucket &, const storage::spi::Bucket &, @@ -261,18 +261,17 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { } RetrieversSP getDocumentRetrievers(storage::spi::ReadConsistency) override { - RetrieversSP ret(new std::vector<IDocumentRetriever::SP>); - ret->push_back(IDocumentRetriever::SP(new MyDocumentRetriever(nullptr, Timestamp(), lastDocId))); - ret->push_back(IDocumentRetriever::SP(new MyDocumentRetriever(document, existingTimestamp, lastDocId))); + auto ret = std::make_shared<std::vector<IDocumentRetriever::SP>>(); + ret->push_back(std::make_shared<MyDocumentRetriever>(nullptr, Timestamp(), lastDocId)); + ret->push_back(std::make_shared<MyDocumentRetriever>(document, existingTimestamp, lastDocId)); return ret; } void handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) override { - BucketIdListResult::List list; - resultHandler.handle(BucketIdListResult(list)); + resultHandler.handle(BucketIdListResult()); } - void handlePopulateActiveBuckets(document::BucketId::List &buckets, IGenericResultHandler &resultHandler) override { + void handlePopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler &resultHandler) override { (void) buckets; resultHandler.handle(Result()); } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index c7a3103af22..37d04f6bcb9 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -225,25 +225,26 @@ BucketDB::deleteEmptyBucket(const BucketId &bucketId) } } -void -BucketDB::getActiveBuckets(BucketId::List &buckets) const +document::BucketId::List +BucketDB::getActiveBuckets() const { + BucketId::List buckets; for (const auto & entry : _map) { if (entry.second.isActive()) { buckets.push_back(entry.first); } } + return buckets; } -void -BucketDB::populateActiveBuckets(const BucketId::List &buckets, BucketId::List &fixupBuckets) +document::BucketId::List +BucketDB::populateActiveBuckets(BucketId::List buckets) { - typedef BucketId::List BIV; - BIV sorted(buckets); - BIV toAdd; - std::sort(sorted.begin(), sorted.end()); - auto si = sorted.begin(); - auto se = sorted.end(); + BucketId::List toAdd; + BucketId::List fixupBuckets; + std::sort(buckets.begin(), buckets.end()); + auto si = buckets.begin(); + auto se = buckets.end(); for (const auto & entry : _map) { for (; si != se && !(entry.first < *si); ++si) { if (*si < entry.first) { @@ -263,6 +264,7 @@ BucketDB::populateActiveBuckets(const BucketId::List &buckets, BucketId::List &f InsertResult ins(_map.emplace(bucketId, activeState)); assert(ins.second); } + return fixupBuckets; } } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index 2ea7594bde1..6fe46d97f46 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -63,8 +63,8 @@ public: void setBucketState(const BucketId &bucketId, bool active); void createBucket(const BucketId &bucketId); void deleteEmptyBucket(const BucketId &bucketId); - void getActiveBuckets(BucketId::List &buckets) const; - void populateActiveBuckets(const BucketId::List &buckets, BucketId::List &fixupBuckets); + BucketId::List getActiveBuckets() const; + BucketId::List populateActiveBuckets(BucketId::List buckets); ConstMapIterator begin() const { return _map.begin(); } ConstMapIterator end() const { return _map.end(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 28234730f7b..2e6a7087b69 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -982,12 +982,9 @@ DocumentMetaStore::updateActiveLids(const BucketId &bucketId, bool active) } void -DocumentMetaStore::populateActiveBuckets(const BucketId::List &buckets) +DocumentMetaStore::populateActiveBuckets(BucketId::List buckets) { - typedef BucketId::List BIV; - BIV fixupBuckets; - - _bucketDB->takeGuard()->populateActiveBuckets(buckets, fixupBuckets); + BucketId::List fixupBuckets = _bucketDB->takeGuard()->populateActiveBuckets(std::move(buckets)); for (const auto &bucketId : fixupBuckets) { updateActiveLids(bucketId, true); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 9e4977c65e1..5cb0b1469fd 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -233,7 +233,7 @@ public: bucketdb::BucketDeltaPair handleSplit(const bucketdb::SplitBucketSession &session) override; bucketdb::BucketDeltaPair handleJoin(const bucketdb::JoinBucketsSession &session) override; void setBucketState(const BucketId &bucketId, bool active) override; - void populateActiveBuckets(const BucketId::List &buckets) override; + void populateActiveBuckets(BucketId::List buckets) override; ConstIterator beginFrozen() const; const vespalib::GenerationHandler & getGenerationHandler() const { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h index bb8c860b56e..1a0288fea84 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_bucket_handler.h @@ -52,7 +52,7 @@ struct IBucketHandler * Sets the bucket state to active, used when adding document db as * part of live reconfig. **/ - virtual void populateActiveBuckets(const BucketId::List &buckets) = 0; + virtual void populateActiveBuckets(BucketId::List buckets) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 7633b5a32c0..4163b59b4ca 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -75,7 +75,7 @@ LidAllocator::unregisterLid(DocId lid) _usedLids.clearBit(lid); if (_activeLids.testBit(lid)) { _activeLids.clearBit(lid); - _numActiveLids = _activeLids.count(); + _numActiveLids.store(_activeLids.count(), std::memory_order_relaxed); } } @@ -90,7 +90,7 @@ LidAllocator::unregister_lids(const std::vector<DocId>& lids) _usedLids.clear_bits(lids); assert(high < _activeLids.size()); _activeLids.consider_clear_bits(lids); - _numActiveLids = _activeLids.count(); + _numActiveLids.store(_activeLids.count(), std::memory_order_relaxed); } void @@ -237,7 +237,7 @@ LidAllocator::updateActiveLids(DocId lid, bool active) } else { _activeLids.clearBit(lid); } - _numActiveLids = _activeLids.count(); + _numActiveLids.store(_activeLids.count(), std::memory_order_relaxed); } } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 947192605e2..08fa4a6489a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -6,6 +6,7 @@ #include "lidstatevector.h" #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/queryeval/blueprint.h> +#include <atomic> namespace proton::documentmetastore { @@ -25,7 +26,7 @@ private: LidStateVector _pendingHoldLids; bool _lidFreeListConstructed; LidStateVector _activeLids; - uint32_t _numActiveLids; + std::atomic<uint32_t> _numActiveLids; public: LidAllocator(uint32_t size, @@ -54,8 +55,8 @@ public: void clearDocs(DocId lidLow, DocId lidLimit); void shrinkLidSpace(DocId committedDocIdLimit); uint32_t getNumUsedLids() const { return _usedLids.count(); } - uint32_t getNumActiveLids() const { - return _numActiveLids; + uint32_t getNumActiveLids() const noexcept { + return _numActiveLids.load(std::memory_order_relaxed); } void setFreeListConstructed() { _lidFreeListConstructed = true; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 8489a68af15..0ec27ac419f 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -177,14 +177,14 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req) } bool MatchEngine::isOnline() const { - return _nodeUp; + return _nodeUp.load(std::memory_order_relaxed); } void MatchEngine::setNodeUp(bool nodeUp) { - _nodeUp = nodeUp; + _nodeUp.store(nodeUp, std::memory_order_relaxed); } void @@ -192,7 +192,7 @@ MatchEngine::setNodeMaintenance(bool nodeMaintenance) { _nodeMaintenance = nodeMaintenance; if (nodeMaintenance) { - _nodeUp = false; + _nodeUp.store(false, std::memory_order_relaxed); } } diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 3d3be775a4a..fcafdc5a5f8 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -25,7 +25,7 @@ private: HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; vespalib::SimpleThreadBundle::Pool _threadBundlePool; - bool _nodeUp; + std::atomic<bool> _nodeUp; bool _nodeMaintenance; public: diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp index 335a3dc70fb..59f44c86d92 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp @@ -34,6 +34,18 @@ AttributeLimiter::AttributeLimiter(Searchable &searchable_attributes, { } +bool +AttributeLimiter::was_used() const +{ + return (_estimatedHits.load(std::memory_order_relaxed) >= 0); +} + +ssize_t +AttributeLimiter::getEstimatedHits() const +{ + return _estimatedHits.load(std::memory_order_relaxed); +} + AttributeLimiter::~AttributeLimiter() = default; namespace { @@ -78,7 +90,7 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool st field.add(FieldSpec(_attribute_name, my_field_id, my_handle)); _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node); _blueprint->fetchPostings(ExecuteInfo::create(strictSearch)); - _estimatedHits = _blueprint->getState().estimate().estHits; + _estimatedHits.store(_blueprint->getState().estimate().estHits, std::memory_order_relaxed); _blueprint->freeze(); } _match_datas.push_back(layout.createMatchData()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h index 59d32bb997d..3e4efbbd371 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h @@ -32,8 +32,8 @@ public: DiversityCutoffStrategy diversityCutoffStrategy); ~AttributeLimiter(); search::queryeval::SearchIterator::UP create_search(size_t want_hits, size_t max_group_size, bool strictSearch); - bool was_used() const { return ((!_match_datas.empty()) || (_blueprint.get() != nullptr)); } - ssize_t getEstimatedHits() const { return _estimatedHits; } + bool was_used() const; + ssize_t getEstimatedHits() const; static DiversityCutoffStrategy toDiversityCutoffStrategy(vespalib::stringref strategy); private: const vespalib::string & toString(DiversityCutoffStrategy strategy); @@ -45,7 +45,7 @@ private: std::mutex _lock; std::vector<search::fef::MatchData::UP> _match_datas; search::queryeval::Blueprint::UP _blueprint; - ssize_t _estimatedHits; + std::atomic<ssize_t> _estimatedHits; double _diversityCutoffFactor; DiversityCutoffStrategy _diversityCutoffStrategy; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h index 4aa8a3f6392..b133d5a5058 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchlib/queryeval/begin_and_end_id.h> -#include <vespa/fastos/dynamiclibrary.h> +#include <vespa/fastos/types.h> #include <mutex> #include <condition_variable> #include <atomic> diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h index b393a85f632..1edecc59136 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h @@ -70,7 +70,7 @@ public: virtual void handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) = 0; - virtual void handlePopulateActiveBuckets(document::BucketId::List &buckets, IGenericResultHandler &resultHandler) = 0; + virtual void handlePopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler &resultHandler) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 7e21e619419..94882bfadf5 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -107,19 +107,21 @@ public: : _bucketSet() { } ~BucketIdListResultHandler() override; - void handle(const BucketIdListResult &result) override { + void + handle(BucketIdListResult result) override { const BucketIdListResult::List &buckets = result.getList(); for (size_t i = 0; i < buckets.size(); ++i) { _bucketSet.insert(buckets[i]); } } - std::unique_ptr<BucketIdListResult> getResult() const { + std::unique_ptr<BucketIdListResult> + getResult() const { BucketIdListResult::List buckets; buckets.reserve(_bucketSet.size()); for (document::BucketId bucketId : _bucketSet) { buckets.push_back(bucketId); } - return std::make_unique<BucketIdListResult>(buckets); + return std::make_unique<BucketIdListResult>(std::move(buckets)); } }; @@ -139,10 +141,10 @@ public: } } ~SynchronizedBucketIdListResultHandler() override; - void handle(const BucketIdListResult &result) override { + void handle(BucketIdListResult result) override { { std::lock_guard<std::mutex> guard(_lock); - BucketIdListResultHandler::handle(result); + BucketIdListResultHandler::handle(std::move(result)); } countDown(); } @@ -283,7 +285,7 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace) const IPersistenceHandler *handler = snap.handlers().get(); handler->handleListBuckets(resultHandler); } - return *resultHandler.getResult(); + return std::move(*resultHandler.getResult()); } @@ -641,10 +643,10 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetModifiedBuckets(resultHandler); } - for (const auto & item : extraModifiedBuckets) { - resultHandler.handle(*item); + for (auto & item : extraModifiedBuckets) { + resultHandler.handle(std::move(*item)); } - return dynamic_cast<BucketIdListResult &>(*futureResult.get()); + return std::move(dynamic_cast<BucketIdListResult &>(*futureResult.get())); } @@ -763,7 +765,7 @@ private: public: ActiveBucketIdListResultHandler() : _bucketMap() { } - void handle(const BucketIdListResult &result) override { + void handle(BucketIdListResult result) override { const BucketIdListResult::List &buckets = result.getList(); for (size_t i = 0; i < buckets.size(); ++i) { IR ir(_bucketMap.insert(std::make_pair(buckets[i], 1u))); @@ -805,7 +807,7 @@ PersistenceEngine::populateInitialBucketDB(const WriteGuard & guard, BucketSpace auto catchResult = std::make_unique<storage::spi::CatchResult>(); auto futureResult = catchResult->future_result(); GenericResultHandler trHandler(1, std::move(catchResult)); - targetHandler.handlePopulateActiveBuckets(buckets, trHandler); + targetHandler.handlePopulateActiveBuckets(std::move(buckets), trHandler); futureResult.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h index 27b1fc8bdd0..897ff75c7b3 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h @@ -9,12 +9,12 @@ template <typename ResultType> class IResultHandler { public: virtual ~IResultHandler() = default; - virtual void handle(const ResultType &result) = 0; + virtual void handle(ResultType result) = 0; }; using IBucketIdListResultHandler = IResultHandler<storage::spi::BucketIdListResult>; -using IBucketInfoResultHandler = IResultHandler<storage::spi::BucketInfoResult>; -using IGenericResultHandler = IResultHandler<storage::spi::Result>; +using IBucketInfoResultHandler = IResultHandler<const storage::spi::BucketInfoResult &>; +using IGenericResultHandler = IResultHandler<const storage::spi::Result &>; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp index 0d9b2f6aaf6..d9201ce5b9e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp @@ -46,15 +46,14 @@ void BucketHandler::performPopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler *resultHandler) { - _ready->populateActiveBuckets(buckets); + _ready->populateActiveBuckets(std::move(buckets)); resultHandler->handle(Result()); } void BucketHandler::deactivateAllActiveBuckets() { - BucketId::List buckets; - _ready->getBucketDB().takeGuard()->getActiveBuckets(buckets); + BucketId::List buckets = _ready->getBucketDB().takeGuard()->getActiveBuckets(); for (auto bucketId : buckets) { _ready->setBucketState(bucketId, storage::spi::BucketInfo::NOT_ACTIVE); // Don't notify bucket state changed, node is marked down so @@ -93,7 +92,7 @@ BucketHandler::handleListBuckets(IBucketIdListResultHandler &resultHandler) // master write thread in document database. BucketIdListResult::List buckets; _ready->getBucketDB().takeGuard()->getBuckets(buckets); - resultHandler.handle(BucketIdListResult(buckets)); + resultHandler.handle(BucketIdListResult(std::move(buckets))); } void @@ -130,17 +129,16 @@ BucketHandler::handleListActiveBuckets(IBucketIdListResultHandler &resultHandler // Called by SPI thread. // BucketDBOwner ensures synchronization between SPI thread and // master write thread in document database. - BucketIdListResult::List buckets; - _ready->getBucketDB().takeGuard()->getActiveBuckets(buckets); - resultHandler.handle(BucketIdListResult(buckets)); + + resultHandler.handle(BucketIdListResult(_ready->getBucketDB().takeGuard()->getActiveBuckets())); } void -BucketHandler::handlePopulateActiveBuckets(document::BucketId::List &buckets, +BucketHandler::handlePopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler &resultHandler) { - _executor.execute(makeLambdaTask([this, buckets, &resultHandler]() { - performPopulateActiveBuckets(std::move(buckets), &resultHandler); + _executor.execute(makeLambdaTask([this, moved_buckets=std::move(buckets), &resultHandler]() mutable { + performPopulateActiveBuckets(std::move(moved_buckets), &resultHandler); })); } diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h index 2344e080450..79c6e929a51 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h @@ -60,7 +60,7 @@ public: void handleGetBucketInfo(const storage::spi::Bucket &bucket, IBucketInfoResultHandler &resultHandler); void handleListActiveBuckets(IBucketIdListResultHandler &resultHandler); - void handlePopulateActiveBuckets(document::BucketId::List &buckets, + void handlePopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler &resultHandler); bool hasBucket(const storage::spi::Bucket &bucket); diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp index 20cd4ced6b2..e7142fb5bde 100644 --- a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp @@ -75,10 +75,7 @@ ClusterStateHandler::performSetClusterState(const ClusterState *calc, IGenericRe void ClusterStateHandler::performGetModifiedBuckets(IBucketIdListResultHandler *resultHandler) { - storage::spi::BucketIdListResult::List modifiedBuckets; - modifiedBuckets.resize(_modifiedBuckets.size()); - std::copy(_modifiedBuckets.begin(), _modifiedBuckets.end(), - modifiedBuckets.begin()); + storage::spi::BucketIdListResult::List modifiedBuckets(_modifiedBuckets.begin(), _modifiedBuckets.end()); if (LOG_WOULD_LOG(debug) && !modifiedBuckets.empty()) { std::ostringstream oss; @@ -91,7 +88,7 @@ ClusterStateHandler::performGetModifiedBuckets(IBucketIdListResultHandler *resul LOG(debug, "performGetModifiedBuckets(): modifiedBuckets(%zu): %s", modifiedBuckets.size(), oss.str().c_str()); } - resultHandler->handle(BucketIdListResult(modifiedBuckets)); + resultHandler->handle(BucketIdListResult(std::move(modifiedBuckets))); _modifiedBuckets.clear(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index e80d8033751..402de8ce7ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -20,6 +20,7 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/searchlib/transactionlog/client_session.h> +#include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> #include <cassert> @@ -42,6 +43,7 @@ using vespalib::make_string; using std::make_unique; using std::make_shared; using search::CommitParam; +using namespace vespalib::atomic; namespace proton { @@ -305,13 +307,13 @@ void FeedHandler::performEof() { assert(_writeService.master().isCurrentThread()); - _activeFeedView->forceCommitAndWait(CommitParam(_serialNum)); + _activeFeedView->forceCommitAndWait(CommitParam(load_relaxed(_serialNum))); LOG(debug, "Visiting done for transaction log domain '%s', eof received", _tlsMgr.getDomainName().c_str()); // Replay must be complete - if (_replay_end_serial_num != _serialNum) { + if (_replay_end_serial_num != load_relaxed(_serialNum)) { LOG(warning, "Expected replay end serial number %" PRIu64 ", got serial number %" PRIu64, - _replay_end_serial_num, _serialNum); - assert(_replay_end_serial_num == _serialNum); + _replay_end_serial_num, load_relaxed(_serialNum)); + assert(_replay_end_serial_num == load_relaxed(_serialNum)); } _owner.onTransactionLogReplayDone(); _tlsMgr.replayDone(); @@ -444,7 +446,7 @@ void FeedHandler::init(SerialNum oldestConfigSerial) { _tlsMgr.init(oldestConfigSerial, _prunedSerialNum, _replay_end_serial_num); - _serialNum = _prunedSerialNum; + store_relaxed(_serialNum, _prunedSerialNum); if (_tlsWriter == nullptr) { _tlsMgrWriter = std::make_unique<TlsMgrWriter>(_tlsMgr, _tlsWriterfactory); _tlsWriter = _tlsMgrWriter.get(); @@ -458,7 +460,7 @@ void FeedHandler::close() { if (_allowSync) { - syncTls(_serialNum); + syncTls(load_relaxed(_serialNum)); } _allowSync = false; _tlsMgr.close(); @@ -484,8 +486,8 @@ FeedHandler::replayTransactionLog(SerialNum flushedIndexMgrSerial, SerialNum flu TransactionLogManager::prepareReplay(_tlsMgr.getClient(), _docTypeName.getName(), flushedIndexMgrSerial, flushedSummaryMgrSerial, config_store); - _tlsReplayProgress = _tlsMgr.make_replay_progress(_serialNum, _replay_end_serial_num); - _tlsMgr.startReplay(_serialNum, _replay_end_serial_num, *this); + _tlsReplayProgress = _tlsMgr.make_replay_progress(load_relaxed(_serialNum), _replay_end_serial_num); + _tlsMgr.startReplay(load_relaxed(_serialNum), _replay_end_serial_num, *this); } void @@ -549,7 +551,7 @@ FeedHandler::initiateCommit(vespalib::steady_time start_time) { if (_activeFeedView) { using KeepAlivePair = vespalib::KeepAlive<std::pair<CommitResult, DoneCallback>>; auto pair = std::make_pair(std::move(commitResult), std::move(onCommitDoneContext)); - _activeFeedView->forceCommit(CommitParam(_serialNum, CommitParam::UpdateStats::SKIP), std::make_shared<KeepAlivePair>(std::move(pair))); + _activeFeedView->forceCommit(CommitParam(load_relaxed(_serialNum), CommitParam::UpdateStats::SKIP), std::make_shared<KeepAlivePair>(std::move(pair))); } } @@ -773,7 +775,7 @@ FeedHandler::heartBeat() { assert(_writeService.master().isCurrentThread()); _heart_beat_time.store(vespalib::steady_clock::now()); - _activeFeedView->heartBeat(_serialNum, vespalib::IDestructorCallback::SP()); + _activeFeedView->heartBeat(load_relaxed(_serialNum), vespalib::IDestructorCallback::SP()); } FeedHandler::RPC::Result diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 32a70f7c2b0..4e9c016af9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -78,7 +78,7 @@ private: TlsWriter *_tlsWriter; TlsReplayProgress::UP _tlsReplayProgress; // the serial num of the last feed operation processed by feed handler. - SerialNum _serialNum; + std::atomic<SerialNum> _serialNum; // the serial num considered to be fully procssessed and flushed to stable storage. Used to prune transaction log. SerialNum _prunedSerialNum; // the serial num of the last feed operation in the transaction log at startup before replay @@ -215,9 +215,15 @@ public: _bucketDBHandler = bucketDBHandler; } - void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; } - SerialNum inc_serial_num() override { return ++_serialNum; } - SerialNum getSerialNum() const override { return _serialNum; } + // Must only be called from writer thread: + void setSerialNum(SerialNum serialNum) { _serialNum.store(serialNum, std::memory_order_relaxed); } + SerialNum inc_serial_num() override { + const auto post_inc = _serialNum.load(std::memory_order_relaxed) + 1u; + _serialNum.store(post_inc, std::memory_order_relaxed); + return post_inc; + } + // May be called from non-writer threads: + SerialNum getSerialNum() const override { return _serialNum.load(std::memory_order_relaxed); } // The two following methods are used when saving initial config SerialNum get_replay_end_serial_num() const { return _replay_end_serial_num; } SerialNum inc_replay_end_serial_num() { return ++_replay_end_serial_num; } diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp index bec9197501b..021460e6ecc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp @@ -133,9 +133,9 @@ PersistenceHandlerProxy::handleListActiveBuckets(IBucketIdListResultHandler &res } void -PersistenceHandlerProxy::handlePopulateActiveBuckets(document::BucketId::List &buckets, IGenericResultHandler &resultHandler) +PersistenceHandlerProxy::handlePopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler &resultHandler) { - _bucketHandler.handlePopulateActiveBuckets(buckets, resultHandler); + _bucketHandler.handlePopulateActiveBuckets(std::move(buckets), resultHandler); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h index 5c35434aae8..c438a777f1b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h @@ -57,7 +57,7 @@ public: void handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) override; - void handlePopulateActiveBuckets(document::BucketId::List &buckets, IGenericResultHandler &resultHandler) override; + void handlePopulateActiveBuckets(document::BucketId::List buckets, IGenericResultHandler &resultHandler) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 170b6f99930..a537742b79b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -451,11 +451,11 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) } else { putSummaryNoop(std::move(futureStream), onWriteDone); } - auto task = makeLambdaTask([upd = updOp.getUpdate(), useDocStore, lid, onWriteDone, + auto task = makeLambdaTask([upd = updOp.getUpdate(), useDocStore, lid, is_replay = onWriteDone->is_replay(), promisedDoc = std::move(promisedDoc), promisedStream = std::move(promisedStream), this]() mutable { - makeUpdatedDocument(useDocStore, lid, *upd, onWriteDone, + makeUpdatedDocument(useDocStore, lid, *upd, is_replay, std::move(promisedDoc), std::move(promisedStream)); }); _writeService.shared().execute(CpuUsage::wrap(std::move(task), CpuUsage::Category::WRITE)); @@ -465,19 +465,19 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) void StoreOnlyFeedView::makeUpdatedDocument(bool useDocStore, Lid lid, const DocumentUpdate & update, - OnOperationDoneType onWriteDone, PromisedDoc promisedDoc, + bool is_replay, PromisedDoc promisedDoc, PromisedStream promisedStream) { Document::UP prevDoc = _summaryAdapter->get(lid, *_repo); Document::UP newDoc; vespalib::nbostream newStream(12345); - assert(onWriteDone->is_replay() || useDocStore); + assert(is_replay || useDocStore); if (useDocStore) { assert(prevDoc); } if (!prevDoc) { // Replaying, document removed later before summary was flushed. - assert(onWriteDone->is_replay()); + assert(is_replay); // If we've passed serial number for flushed index then we could // also check that this operation is marked for ignore by index // proxy. @@ -491,7 +491,7 @@ StoreOnlyFeedView::makeUpdatedDocument(bool useDocStore, Lid lid, const Document } else { // Replaying, document removed and lid reused before summary // was flushed. - assert(onWriteDone->is_replay() && !useDocStore); + assert(is_replay && !useDocStore); } } promisedDoc.set_value(std::move(newDoc)); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index bd8509fa796..2822aa70525 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -184,7 +184,7 @@ private: IPendingLidTracker::Token get_pending_lid_token(const DocumentOperation &op); - void makeUpdatedDocument(bool useDocStore, Lid lid, const DocumentUpdate & update, OnOperationDoneType onWriteDone, + void makeUpdatedDocument(bool useDocStore, Lid lid, const DocumentUpdate & update, bool is_replay, PromisedDoc promisedDoc, PromisedStream promisedStream); protected: diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index abc8c8450e4..3a2e443bfef 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchcore_test STATIC mock_index_manager.cpp mock_shared_threading_service.cpp userdocumentsbuilder.cpp + resulthandler.cpp threading_service_observer.cpp transport_helper.cpp DEPENDS diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h index d8803fd9c27..c773ae32b3f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h +++ b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h @@ -5,6 +5,7 @@ #include <vespa/document/bucket/bucketidlist.h> #include <vespa/document/bucket/bucket.h> #include <set> +#include <memory> namespace proton::test { @@ -23,8 +24,8 @@ private: bool _nodeMaintenance; public: - typedef std::shared_ptr<BucketStateCalculator> SP; - BucketStateCalculator() : + using SP = std::shared_ptr<BucketStateCalculator>; + BucketStateCalculator() noexcept : _ready(), _asked(), _clusterUp(true), @@ -33,6 +34,8 @@ public: _nodeMaintenance(false) { } + BucketStateCalculator(BucketStateCalculator &&) noexcept = default; + BucketStateCalculator & operator =(BucketStateCalculator &&) noexcept = default; ~BucketStateCalculator() override; BucketStateCalculator &addReady(const document::BucketId &bucket) { _ready.insert(bucket); diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h index 0386cb141ee..b9dacf61af8 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h @@ -115,8 +115,8 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore void setBucketState(const BucketId &bucketId, bool active) override { _store.setBucketState(bucketId, active); } - void populateActiveBuckets(const document::BucketId::List &buckets) override { - _store.populateActiveBuckets(buckets); + void populateActiveBuckets(document::BucketId::List buckets) override { + _store.populateActiveBuckets(std::move(buckets)); } diff --git a/searchcore/src/vespa/searchcore/proton/test/resulthandler.cpp b/searchcore/src/vespa/searchcore/proton/test/resulthandler.cpp new file mode 100644 index 00000000000..00102aeab25 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/resulthandler.cpp @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "resulthandler.h" + +namespace proton::test { + +GenericResultHandler::~GenericResultHandler() = default; + +void +GenericResultHandler::handle(const storage::spi::Result &result) { + _result = std::make_unique<storage::spi::Result>(result); +} + +BucketInfoResultHandler::~BucketInfoResultHandler() = default; +void +BucketInfoResultHandler::handle(const storage::spi::BucketInfoResult &result) { + _result = std::make_unique<storage::spi::BucketInfoResult>(result); +} + +BucketIdListResultHandler::~BucketIdListResultHandler() = default; + +void +BucketIdListResultHandler::handle(storage::spi::BucketIdListResult result) { + _result = std::make_unique<storage::spi::BucketIdListResult>(std::move(result)); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/test/resulthandler.h b/searchcore/src/vespa/searchcore/proton/test/resulthandler.h index 00896cb84fe..d5954e30678 100644 --- a/searchcore/src/vespa/searchcore/proton/test/resulthandler.h +++ b/searchcore/src/vespa/searchcore/proton/test/resulthandler.h @@ -3,19 +3,16 @@ #include <vespa/searchcore/proton/persistenceengine/resulthandler.h> -namespace proton { - -namespace test { +namespace proton::test { class GenericResultHandler : public IGenericResultHandler { private: std::unique_ptr<storage::spi::Result> _result; public: - void handle(const storage::spi::Result &result) override { - _result.reset(new storage::spi::Result(result)); - } - bool valid() const { return _result.get() != NULL; } + ~GenericResultHandler() override; + void handle(const storage::spi::Result &result) override; + bool valid() const { return static_cast<bool>(_result); } const storage::spi::Result &getResult() const { return *_result; } }; @@ -25,10 +22,9 @@ class BucketInfoResultHandler : public IBucketInfoResultHandler private: std::unique_ptr<storage::spi::BucketInfoResult> _result; public: - void handle(const storage::spi::BucketInfoResult &result) override { - _result.reset(new storage::spi::BucketInfoResult(result)); - } - bool valid() const { return _result.get() != NULL; } + ~BucketInfoResultHandler() override; + void handle(const storage::spi::BucketInfoResult &result) override; + bool valid() const { return static_cast<bool>(_result); } const storage::spi::BucketInfoResult &getResult() const { return *_result; } const storage::spi::BucketInfo &getInfo() const { return getResult().getBucketInfo(); } }; @@ -39,16 +35,11 @@ class BucketIdListResultHandler : public IBucketIdListResultHandler private: std::unique_ptr<storage::spi::BucketIdListResult> _result; public: - void handle(const storage::spi::BucketIdListResult &result) override { - _result.reset(new storage::spi::BucketIdListResult(result)); - } - bool valid() const { return _result.get() != NULL; } + ~BucketIdListResultHandler() override; + void handle(storage::spi::BucketIdListResult result) override; + bool valid() const { return static_cast<bool>(_result); } const storage::spi::BucketIdListResult &getResult() const { return *_result; } const storage::spi::BucketIdListResult::List &getList() const { return getResult().getList(); } }; - -} // namespace test - -} // namespace proton - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 20aba93d4d7..2e53db3bd87 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -490,7 +490,7 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index) args->old_absolute_id = _current_index_id + _last_fusion_id; args->old_source_list = _source_list; string selector_name = IndexDiskLayout::getSelectorFileName(getFlushDir(args->old_absolute_id)); - args->flush_serial_num = _current_serial_num; + args->flush_serial_num = current_serial_num(); { LockGuard lock(_index_update_lock); // Handover of extra memory indexes to flush @@ -795,7 +795,7 @@ IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex) args._oldSourceList = _source_list; // Delay destruction uint32_t oldAbsoluteId = _current_index_id + _last_fusion_id; string selectorName = IndexDiskLayout::getSelectorFileName(getFlushDir(oldAbsoluteId)); - SerialNum freezeSerialNum = _current_serial_num; + SerialNum freezeSerialNum = current_serial_num(); bool dropEmptyLast = false; SaveInfo::UP saveInfo; @@ -921,7 +921,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, _flush_serial_num = IndexReadUtilities::readSerialNum(latest_index_dir); _lastFlushTime = search::FileKit::getModificationTime(latest_index_dir); - _current_serial_num = _flush_serial_num; + set_current_serial_num(_flush_serial_num); const string selector = IndexDiskLayout::getSelectorFileName(latest_index_dir); _selector = FixedSourceSelector::load(selector, _next_id - 1); } else { @@ -941,7 +941,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, assert(_current_index_id < ISourceSelector::SOURCE_LIMIT); _selector->setDefaultSource(_current_index_id); auto sourceList = loadDiskIndexes(spec, std::make_unique<IndexCollection>(_selector)); - _current_index = operations.createMemoryIndex(_schema, *sourceList, _current_serial_num); + _current_index = operations.createMemoryIndex(_schema, *sourceList, current_serial_num()); LOG(debug, "Index manager created with flushed serial num %" PRIu64, _flush_serial_num); sourceList->append(_current_index_id, _current_index); sourceList->setCurrentIndex(_current_index_id); @@ -966,10 +966,10 @@ IndexMaintainer::initFlush(SerialNum serialNum, searchcorespi::FlushStats * stat assert(_ctx.getThreadingService().master().isCurrentThread()); // while flush engine scheduler thread waits { LockGuard lock(_index_update_lock); - _current_serial_num = std::max(_current_serial_num, serialNum); + set_current_serial_num(std::max(current_serial_num(), serialNum)); } - IMemoryIndex::SP new_index(_operations.createMemoryIndex(getSchema(), *_current_index, _current_serial_num)); + IMemoryIndex::SP new_index(_operations.createMemoryIndex(getSchema(), *_current_index, current_serial_num())); FlushArgs args; args.stats = stats; // Ensure that all index thread tasks accessing memory index have completed. @@ -984,7 +984,7 @@ IndexMaintainer::initFlush(SerialNum serialNum, searchcorespi::FlushStats * stat if (args._skippedEmptyLast && args._extraIndexes.empty()) { // No memory index to flush, it was empty LockGuard lock(_state_lock); - _flush_serial_num = _current_serial_num; + _flush_serial_num = current_serial_num(); _lastFlushTime = vespalib::system_clock::now(); LOG(debug, "No memory index to flush. Update serial number and flush time to current: " "flushSerialNum(%" PRIu64 "), lastFlushTime(%f)", @@ -1014,7 +1014,7 @@ IndexMaintainer::doFusion(SerialNum serialNum, std::shared_ptr<search::IFlushTok // XXX: Claims to have flushed memory index when starting fusion. { LockGuard lock(_index_update_lock); - _current_serial_num = std::max(_current_serial_num, serialNum); + set_current_serial_num(std::max(current_serial_num(), serialNum)); } FusionSpec spec; @@ -1234,7 +1234,7 @@ IndexMaintainer::putDocument(uint32_t lid, const Document &doc, SerialNum serial _selector->setSource(lid, _current_index_id); _source_list->setSource(lid); ++_source_selector_changes; - _current_serial_num = serialNum; + set_current_serial_num(serialNum); } void @@ -1247,7 +1247,7 @@ IndexMaintainer::removeDocuments(LidVector lids, SerialNum serialNum) _source_list->setSource(lid); } _source_selector_changes += lids.size(); - _current_serial_num = serialNum; + set_current_serial_num(serialNum); _current_index->removeDocuments(std::move(lids)); } @@ -1267,7 +1267,7 @@ IndexMaintainer::commit(vespalib::Gate& gate) // only triggered via commit_and_wait() assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); - _current_index->commit(std::make_shared<vespalib::GateCallback>(gate), _current_serial_num); + _current_index->commit(std::make_shared<vespalib::GateCallback>(gate), current_serial_num()); } void @@ -1275,7 +1275,7 @@ IndexMaintainer::commit(SerialNum serialNum, OnWriteDoneType onWriteDone) { assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); - _current_serial_num = serialNum; + set_current_serial_num(serialNum); _current_index->commit(onWriteDone, serialNum); } @@ -1284,7 +1284,7 @@ IndexMaintainer::heartBeat(SerialNum serialNum) { assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); - _current_serial_num = serialNum; + set_current_serial_num(serialNum); } void @@ -1293,7 +1293,7 @@ IndexMaintainer::compactLidSpace(uint32_t lidLimit, SerialNum serialNum) assert(_ctx.getThreadingService().index().isCurrentThread()); LOG(info, "compactLidSpace(%u, %" PRIu64 ")", lidLimit, serialNum); LockGuard lock(_index_update_lock); - _current_serial_num = serialNum; + set_current_serial_num(serialNum); _selector->compactLidSpace(lidLimit); } @@ -1313,7 +1313,7 @@ IndexMaintainer::setSchema(const Schema & schema, SerialNum serialNum) { assert(_ctx.getThreadingService().master().isCurrentThread()); pruneRemovedFields(schema, serialNum); - IMemoryIndex::SP new_index(_operations.createMemoryIndex(schema, *_current_index, _current_serial_num)); + IMemoryIndex::SP new_index(_operations.createMemoryIndex(schema, *_current_index, current_serial_num())); SetSchemaArgs args; args._newSchema = schema; diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index b4bba209937..dfae5b4d643 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -18,6 +18,7 @@ #include <vespa/searchcorespi/flush/flushstats.h> #include <vespa/searchlib/attribute/fixedsourceselector.h> #include <vespa/searchlib/common/serialnum.h> +#include <atomic> namespace document { class Document; } @@ -87,14 +88,14 @@ class IndexMaintainer : public IIndexManager, // _selector is protected by SL + IUL ISourceSelector::SP _selector; ISearchableIndexCollection::SP _source_list; // Protected by SL + NSL, only set by master thread - uint32_t _last_fusion_id; // Protected by SL + IUL - uint32_t _next_id; // Protected by SL + IUL - uint32_t _current_index_id; // Protected by SL + IUL - IMemoryIndex::SP _current_index; // Protected by SL + IUL - bool _flush_empty_current_index; - SerialNum _current_serial_num;// Protected by IUL - SerialNum _flush_serial_num; // Protected by SL - vespalib::system_time _lastFlushTime; // Protected by SL + uint32_t _last_fusion_id; // Protected by SL + IUL + uint32_t _next_id; // Protected by SL + IUL + uint32_t _current_index_id; // Protected by SL + IUL + IMemoryIndex::SP _current_index; // Protected by SL + IUL + bool _flush_empty_current_index; + std::atomic<SerialNum> _current_serial_num;// Protected by IUL + SerialNum _flush_serial_num; // Protected by SL + vespalib::system_time _lastFlushTime; // Protected by SL // Extra frozen memory indexes. This list is empty unless new // memory index has been added by force (due to config change or // data structure limitations). @@ -263,6 +264,12 @@ class IndexMaintainer : public IIndexManager, void commit_and_wait(); void commit(vespalib::Gate& gate); void pruneRemovedFields(const Schema &schema, SerialNum serialNum); + [[nodiscard]] SerialNum current_serial_num() const noexcept { + return _current_serial_num.load(std::memory_order_relaxed); + } + void set_current_serial_num(SerialNum new_serial_num) noexcept { + _current_serial_num.store(new_serial_num, std::memory_order_relaxed); + } public: IndexMaintainer(const IndexMaintainer &) = delete; @@ -270,7 +277,7 @@ public: IndexMaintainer(const IndexMaintainerConfig &config, const IndexMaintainerContext &context, IIndexMaintainerOperations &operations); - ~IndexMaintainer(); + ~IndexMaintainer() override; /** * Starts a new MemoryIndex, and dumps the previous one to disk. @@ -333,7 +340,7 @@ public: void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override; SerialNum getCurrentSerialNum() const override { - return _current_serial_num; + return _current_serial_num.load(std::memory_order_relaxed); } SerialNum getFlushedSerialNum() const override { diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index c360bad6d75..02ff01043b0 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/searchlib/attribute/enum_store_loaders.h> #include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 29af989d484..566001cbe17 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -80,6 +80,7 @@ protected: using generation_t = vespalib::GenerationHandler::generation_t; public: + using ArrayRef = vespalib::ArrayRef<EntryT>; using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; MappingTestBase() : _stats(), @@ -107,8 +108,8 @@ public: ~MappingTestBase() { } void set(uint32_t docId, const std::vector<EntryT> &values) { _mvMapping->set(docId, values); } - void replace(uint32_t docId, const std::vector<EntryT> &values) { _mvMapping->replace(docId, values); } ConstArrayRef get(uint32_t docId) { return _mvMapping->get(docId); } + ArrayRef get_writable(uint32_t docId) { return _mvMapping->get_writable(docId); } void assertGet(uint32_t docId, const std::vector<EntryT> &exp) { ConstArrayRef act = get(docId); EXPECT_EQ(exp, std::vector<EntryT>(act.cbegin(), act.cend())); @@ -307,7 +308,7 @@ TEST_F(IntMappingTest, test_that_totalValueCnt_works) EXPECT_EQ(5u, getTotalValueCnt()); } -TEST_F(IntMappingTest, test_that_replace_works) +TEST_F(IntMappingTest, test_that_get_writable_works) { setup(3); addDocs(10); @@ -315,7 +316,12 @@ TEST_F(IntMappingTest, test_that_replace_works) auto old4 = get(4); assertArray({10, 14, 17, 16}, old4); EXPECT_EQ(4u, getTotalValueCnt()); - replace(4, {20, 24, 27, 26}); + { + auto array = get_writable(4); + for (auto& elem : array) { + elem += 10; + } + } assertArray({20, 24, 27, 26}, old4); EXPECT_EQ(4u, getTotalValueCnt()); } diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp index d580e361bff..b95478072de 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("featureoverride_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/fef/fef.h> @@ -9,11 +7,23 @@ LOG_SETUP("featureoverride_test"); #include <vespa/searchlib/fef/test/plugin/double.h> #include <vespa/searchlib/fef/test/plugin/sum.h> #include <vespa/searchlib/features/valuefeature.h> +#include <vespa/searchlib/features/rankingexpressionfeature.h> +#include <vespa/searchlib/fef/test/test_features.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/issue.h> -using namespace search::fef; -using namespace search::fef::test; using namespace search::features; +using namespace search::fef::test; +using namespace search::fef; using search::feature_t; +using vespalib::Issue; +using vespalib::eval::TensorSpec; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::make_string_short::fmt; + typedef Blueprint::SP BPSP; @@ -50,7 +60,7 @@ TEST_F("test decorator - single override", Fixture) { FeatureExecutor *fe = &f.createValueExecutor(); vespalib::Stash &stash = f.stash; - fe = &stash.create<FeatureOverrider>(*fe, 1, 50.0); + fe = &stash.create<FeatureOverrider>(*fe, 1, 50.0, nullptr); f.add(fe, 3).run(); EXPECT_EQUAL(fe->outputs().size(), 3u); @@ -63,8 +73,8 @@ TEST_F("test decorator - multiple overrides", Fixture) { FeatureExecutor *fe = &f.createValueExecutor(); vespalib::Stash &stash = f.stash; - fe = &stash.create<FeatureOverrider>(*fe, 0, 50.0); - fe = &stash.create<FeatureOverrider>(*fe, 2, 100.0); + fe = &stash.create<FeatureOverrider>(*fe, 0, 50.0, nullptr); + fe = &stash.create<FeatureOverrider>(*fe, 2, 100.0, nullptr); f.add(fe, 3).run(); EXPECT_EQUAL(fe->outputs().size(), 3u); @@ -77,7 +87,7 @@ TEST_F("test decorator - non-existing override", Fixture) { FeatureExecutor *fe = &f.createValueExecutor(); vespalib::Stash &stash = f.stash; - fe = &stash.create<FeatureOverrider>(*fe, 1000, 50.0); + fe = &stash.create<FeatureOverrider>(*fe, 1000, 50.0, nullptr); f.add(fe, 3).run(); EXPECT_EQUAL(fe->outputs().size(), 3u); @@ -90,12 +100,12 @@ TEST_F("test decorator - transitive override", Fixture) { FeatureExecutor *fe = &f.createValueExecutor(); vespalib::Stash &stash = f.stash; - fe = &stash.create<FeatureOverrider>(*fe, 1, 50.0); + fe = &stash.create<FeatureOverrider>(*fe, 1, 50.0, nullptr); f.add(fe, 3); EXPECT_EQUAL(fe->outputs().size(), 3u); FeatureExecutor *fe2 = &stash.create<DoubleExecutor>(3); - fe2 = &stash.create<FeatureOverrider>(*fe2, 2, 10.0); + fe2 = &stash.create<FeatureOverrider>(*fe2, 2, 10.0, nullptr); auto inputs = stash.create_array<LazyValue>(3, nullptr); inputs[0] = LazyValue(fe->outputs().get_raw(0), fe); inputs[1] = LazyValue(fe->outputs().get_raw(1), fe); @@ -169,4 +179,123 @@ TEST("test overrides") EXPECT_APPROX(res["double(value(3)).0"], 6.0, 1e-6); } +//----------------------------------------------------------------------------- + +struct SimpleRankFixture { + BlueprintFactory factory; + IndexEnvironment indexEnv; + BlueprintResolver::SP resolver; + Properties overrides; + MatchData::UP match_data; + RankProgram program; + static vespalib::string expr_feature(const vespalib::string &name) { + return fmt("rankingExpression(%s)", name.c_str()); + } + SimpleRankFixture() + : factory(), indexEnv(), resolver(new BlueprintResolver(factory, indexEnv)), + overrides(), match_data(), program(resolver) + { + factory.addPrototype(std::make_shared<DocidBlueprint>()); + factory.addPrototype(std::make_shared<RankingExpressionBlueprint>()); + } + ~SimpleRankFixture(); + void add_expr(const vespalib::string &name, const vespalib::string &expr) { + vespalib::string feature_name = expr_feature(name); + vespalib::string expr_name = feature_name + ".rankingScript"; + indexEnv.getProperties().add(expr_name, expr); + } + void add_override(const vespalib::string &name, const TensorSpec &spec) { + vespalib::nbostream data; + auto tensor = vespalib::eval::value_from_spec(spec, FastValueBuilderFactory::get()); + vespalib::eval::encode_value(*tensor, data); + overrides.add(name, vespalib::stringref(data.peek(), data.size())); + } + void add_override(const vespalib::string &name, const vespalib::string &str) { + overrides.add(name, str); + } + bool try_compile(const vespalib::string &seed) { + resolver->addSeed(seed); + if (!resolver->compile()) { + return false; + } + MatchDataLayout mdl; + QueryEnvironment queryEnv(&indexEnv); + match_data = mdl.createMatchData(); + program.setup(*match_data, queryEnv, overrides); + return true; + } + void compile(const vespalib::string &seed) { + ASSERT_TRUE(try_compile(seed)); + } + TensorSpec get(uint32_t docid) { + auto result = program.get_seeds(false); + ASSERT_EQUAL(1u, result.num_features()); + return TensorSpec::from_value(result.resolve(0).as_object(docid)); + } +}; +SimpleRankFixture::~SimpleRankFixture() = default; + +TensorSpec from_expr(const vespalib::string &expr) { + auto result = TensorSpec::from_expr(expr); + ASSERT_TRUE(result.type() != "error"); + return result; +} + +struct MyIssues : Issue::Handler { + std::vector<vespalib::string> list; + Issue::Binding capture; + MyIssues() : list(), capture(Issue::listen(*this)) {} + void handle(const Issue &issue) override { list.push_back(issue.message()); } +}; + +//----------------------------------------------------------------------------- + +TEST_F("require expression without override works", SimpleRankFixture) { + auto expect = from_expr("tensor<float>(x[3]):[1,2,3]"); + f1.add_expr("foo", "tensor<float>(x[3]):[1,2,3]"); + f1.compile(f1.expr_feature("foo")); + EXPECT_EQUAL(f1.get(1), expect); +} + +TEST_F("require that const binary override works", SimpleRankFixture) { + auto expect = from_expr("tensor<float>(x[3]):[5,6,7]"); + f1.add_expr("foo", "tensor<float>(x[3]):[1,2,3]"); + f1.add_override(f1.expr_feature("foo"), expect); + f1.compile(f1.expr_feature("foo")); + EXPECT_EQUAL(f1.get(1), expect); +} + +TEST_F("require that non-const binary override works", SimpleRankFixture) { + auto expect = from_expr("tensor<float>(x[3]):[5,6,7]"); + f1.add_expr("foo", "tensor<float>(x[3]):[docid,2,3]"); + f1.add_override(f1.expr_feature("foo"), expect); + f1.compile(f1.expr_feature("foo")); + EXPECT_EQUAL(f1.get(1), expect); +} + +TEST_F("require that wrong type binary override is ignored", SimpleRankFixture) { + MyIssues issues; + auto expect = from_expr("tensor<float>(x[3]):[1,2,3]"); + auto other = from_expr("tensor(x[3]):[5,6,7]"); + f1.add_expr("foo", "tensor<float>(x[3]):[1,2,3]"); + f1.add_override(f1.expr_feature("foo"), other); + f1.compile(f1.expr_feature("foo")); + ASSERT_EQUAL(issues.list.size(), 1u); + EXPECT_LESS(issues.list[0].find("has invalid type"), issues.list[0].size()); + fprintf(stderr, "issue: %s\n", issues.list[0].c_str()); +} + +TEST_F("require that bad format binary override is ignored", SimpleRankFixture) { + MyIssues issues; + auto expect = from_expr("tensor<float>(x[3]):[1,2,3]"); + f1.add_expr("foo", "tensor<float>(x[3]):[1,2,3]"); + f1.add_override(f1.expr_feature("foo"), vespalib::string("bad format")); + f1.compile(f1.expr_feature("foo")); + ASSERT_EQUAL(issues.list.size(), 1u); + EXPECT_LESS(issues.list[0].find("has invalid format"), issues.list[0].size()); + fprintf(stderr, "issue: %s\n", issues.list[0].c_str()); +} + +//----------------------------------------------------------------------------- + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/aggregation/group.h b/searchlib/src/vespa/searchlib/aggregation/group.h index 202c5085133..53061b1447b 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.h +++ b/searchlib/src/vespa/searchlib/aggregation/group.h @@ -5,7 +5,6 @@ #include "aggregationresult.h" #include <vespa/searchlib/common/hitrank.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/fastos/dynamiclibrary.h> #include <vector> namespace search::aggregation { diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 9e5a8d4dfbb..4fff7cf96c1 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -50,6 +50,7 @@ vespa_add_library(searchlib_attribute OBJECT flagattribute.cpp floatbase.cpp i_document_weight_attribute.cpp + i_enum_store.cpp iattributemanager.cpp iattributesavetarget.cpp imported_attribute_vector.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/atomic_utils.h b/searchlib/src/vespa/searchlib/attribute/atomic_utils.h new file mode 100644 index 00000000000..48914de8942 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/atomic_utils.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::datastore { + +class AtomicEntryRef; +class EntryRef; + +} + +namespace search::attribute::atomic_utils { + +/* + * Helper class to map from atomic value to non-atomic value, e.g. + * from AtomicEntryRef to EntryRef. + */ +template <typename MaybeAtomicValue> +class NonAtomicValue { +public: + using type = MaybeAtomicValue; +}; + +template <> +class NonAtomicValue<vespalib::datastore::AtomicEntryRef> +{ +public: + using type = vespalib::datastore::EntryRef; +}; + +template <class MaybeAtomicValue> +using NonAtomicValue_t = typename NonAtomicValue<MaybeAtomicValue>::type; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/defines.h b/searchlib/src/vespa/searchlib/attribute/defines.h index 1588bfb4455..d6e2e3d2555 100644 --- a/searchlib/src/vespa/searchlib/attribute/defines.h +++ b/searchlib/src/vespa/searchlib/attribute/defines.h @@ -5,7 +5,7 @@ #define ENUM_ATTRIBUTE(B) EnumAttribute<B> #define MULTIVALUE_ARG(T) multivalue::Value<T> -#define MULTIVALUE_ENUM_ARG multivalue::Value<IEnumStore::Index> +#define MULTIVALUE_ENUM_ARG multivalue::Value<vespalib::datastore::AtomicEntryRef> #define WEIGHTED_MULTIVALUE_ARG(T) multivalue::WeightedValue<T> -#define WEIGHTED_MULTIVALUE_ENUM_ARG multivalue::WeightedValue<IEnumStore::Index> +#define WEIGHTED_MULTIVALUE_ENUM_ARG multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef> diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp index 7f44dbff5cb..eeaa3e9539f 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp @@ -76,6 +76,8 @@ EnumeratedLoader::EnumeratedLoader(IEnumStore& store) { } +EnumeratedLoader::~EnumeratedLoader() = default; + void EnumeratedLoader::set_ref_counts() { @@ -118,7 +120,7 @@ EnumeratedPostingsLoader::set_ref_count(Index idx, uint32_t ref_count) vespalib::ArrayRef<vespalib::datastore::EntryRef> EnumeratedPostingsLoader::initialize_empty_posting_indexes() { - vespalib::Array<EntryRef>(_indexes.size(), EntryRef()).swap(_posting_indexes); + EntryRefVector(_indexes.size(), EntryRef()).swap(_posting_indexes); return _posting_indexes; } @@ -128,7 +130,7 @@ EnumeratedPostingsLoader::build_dictionary() attribute::LoadedEnumAttributeVector().swap(_loaded_enums); _store.get_dictionary().build_with_payload(_indexes, _posting_indexes); release_enum_indexes(); - vespalib::Array<EntryRef>().swap(_posting_indexes); + EntryRefVector().swap(_posting_indexes); } } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h index b0d3090b92e..2a72fcac628 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h @@ -4,7 +4,6 @@ #include "enum_store_types.h" #include "loadedenumvalue.h" -#include <vespa/vespalib/datastore/entryref.h> namespace search { class IEnumStore; } @@ -22,6 +21,10 @@ protected: void release_enum_indexes(); public: EnumeratedLoaderBase(IEnumStore& store); + EnumeratedLoaderBase(const EnumeratedLoaderBase &) = delete; + EnumeratedLoaderBase & operator =(const EnumeratedLoaderBase &) = delete; + EnumeratedLoaderBase(EnumeratedLoaderBase &&) = delete; + EnumeratedLoaderBase & operator =(EnumeratedLoaderBase &&) = delete; ~EnumeratedLoaderBase(); const IndexVector& get_enum_indexes() const { return _indexes; } const EnumVector& get_enum_value_remapping() const noexcept { return _enum_value_remapping; } @@ -40,6 +43,11 @@ private: public: EnumeratedLoader(IEnumStore& store); + EnumeratedLoader(const EnumeratedLoader &) = delete; + EnumeratedLoader & operator =(const EnumeratedLoader &) = delete; + EnumeratedLoader(EnumeratedLoader &&) = delete; + EnumeratedLoader & operator =(EnumeratedLoader &&) = delete; + ~EnumeratedLoader(); EnumVector& get_enums_histogram() { return _enums_histogram; } void allocate_enums_histogram() { EnumVector(_indexes.size(), 0).swap(_enums_histogram); @@ -54,12 +62,17 @@ public: class EnumeratedPostingsLoader : public EnumeratedLoaderBase { private: using EntryRef = vespalib::datastore::EntryRef; + using EntryRefVector = std::vector<EntryRef, vespalib::allocator_large<EntryRef>>; attribute::LoadedEnumAttributeVector _loaded_enums; - vespalib::Array<EntryRef> _posting_indexes; + EntryRefVector _posting_indexes; bool _has_btree_dictionary; public: EnumeratedPostingsLoader(IEnumStore& store); + EnumeratedPostingsLoader(const EnumeratedPostingsLoader &) = delete; + EnumeratedPostingsLoader & operator =(const EnumeratedPostingsLoader &) = delete; + EnumeratedPostingsLoader(EnumeratedPostingsLoader &&) = delete; + EnumeratedPostingsLoader & operator =(EnumeratedPostingsLoader &&) = delete; ~EnumeratedPostingsLoader(); attribute::LoadedEnumAttributeVector& get_loaded_enums() { return _loaded_enums; } void reserve_loaded_enums(size_t num_values) { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_types.h b/searchlib/src/vespa/searchlib/attribute/enum_store_types.h index d3ce925a41e..435e94fe592 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_types.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_types.h @@ -2,16 +2,16 @@ #pragma once -#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/vespalib/datastore/entryref.h> -#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/allocator.h> +#include <vector> namespace search::enumstore { using Index = vespalib::datastore::EntryRef; using InternalIndex = vespalib::datastore::EntryRefT<22>; -using IndexVector = vespalib::Array<Index>; -using EnumHandle = attribute::IAttributeVector::EnumHandle; -using EnumVector = vespalib::Array<uint32_t>; +using IndexVector = std::vector<Index, vespalib::allocator_large<Index>>; +using EnumHandle = uint32_t; +using EnumVector = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp index 26ff891fbb2..7583200806d 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "enumhintsearchcontext.h" +#include "i_enum_store_dictionary.h" #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/vespalib/datastore/i_unique_store_dictionary_read_snapshot.h> + namespace search::attribute { using queryeval::SearchIterator; -using vespalib::btree::BTreeNode; using fef::TermFieldMatchData; EnumHintSearchContext:: diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h index a1389c1381e..7042de9ddb8 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h @@ -2,15 +2,15 @@ #pragma once -#include "i_enum_store_dictionary.h" #include "ipostinglistsearchcontext.h" -#include <vespa/searchlib/queryeval/searchiterator.h> namespace vespalib::datastore { -class EntryComparator; -class IUniqueStoreDictionaryReadSnapshot; + class EntryComparator; + class IUniqueStoreDictionaryReadSnapshot; } +namespace search { class IEnumStoreDictionary; } + namespace search::attribute { /** @@ -34,7 +34,7 @@ protected: void lookupTerm(const vespalib::datastore::EntryComparator &comp); void lookupRange(const vespalib::datastore::EntryComparator &low, const vespalib::datastore::EntryComparator &high); - queryeval::SearchIterator::UP + std::unique_ptr<queryeval::SearchIterator> createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; void fetchPostings(const queryeval::ExecuteInfo & execInfo) override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 5b47d42b6ed..52f42ed368e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -4,7 +4,6 @@ #include "enum_store_compaction_spec.h" #include "enum_store_dictionary.h" -#include "enum_store_loaders.h" #include "enumcomparator.h" #include "i_enum_store.h" #include "loadedenumvalue.h" diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.cpp b/searchlib/src/vespa/searchlib/attribute/i_enum_store.cpp new file mode 100644 index 00000000000..106f67cede7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.cpp @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "i_enum_store.h" +#include "enum_store_loaders.h" + +namespace search { + +enumstore::EnumeratedLoader +IEnumStore::make_enumerated_loader() { + return enumstore::EnumeratedLoader(*this); +} + +enumstore::EnumeratedPostingsLoader +IEnumStore::make_enumerated_postings_loader() { + return enumstore::EnumeratedPostingsLoader(*this); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index b4e86e2a60c..d8b09ba211f 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -2,28 +2,28 @@ #pragma once -#include "enum_store_loaders.h" #include "enum_store_types.h" #include <vespa/vespalib/datastore/atomic_entry_ref.h> -#include <vespa/vespalib/datastore/unique_store_enumerator.h> +#include <memory> namespace vespalib { - -class AddressSpace; -class MemoryUsage; - + class AddressSpace; + class MemoryUsage; } namespace vespalib::datastore { - -class CompactionSpec; -class CompactionStrategy; -class DataStoreBase; - -template <typename> class UniqueStoreRemapper; - + class CompactionSpec; + class CompactionStrategy; + class DataStoreBase; + class EntryComparator; + template <typename> class UniqueStoreRemapper; + template <typename> class UniqueStoreEnumerator; } +namespace search::enumstore { + class EnumeratedLoader; + class EnumeratedPostingsLoader; +} namespace search { class BufferWriter; @@ -42,7 +42,7 @@ public: using EnumHandle = enumstore::EnumHandle; using EnumVector = enumstore::EnumVector; using EnumIndexRemapper = vespalib::datastore::UniqueStoreRemapper<InternalIndex>; - using Enumerator = vespalib::datastore::UniqueStoreEnumerator<IEnumStore::InternalIndex>; + using Enumerator = vespalib::datastore::UniqueStoreEnumerator<InternalIndex>; using IndexList = std::vector<Index>; @@ -68,13 +68,8 @@ public: // Should only be used by unit tests. virtual void inc_compaction_count() = 0; - enumstore::EnumeratedLoader make_enumerated_loader() { - return enumstore::EnumeratedLoader(*this); - } - - enumstore::EnumeratedPostingsLoader make_enumerated_postings_loader() { - return enumstore::EnumeratedPostingsLoader(*this); - } + enumstore::EnumeratedLoader make_enumerated_loader(); + enumstore::EnumeratedPostingsLoader make_enumerated_postings_loader(); virtual std::unique_ptr<Enumerator> make_enumerator() const = 0; virtual std::unique_ptr<vespalib::datastore::EntryComparator> allocate_comparator() const = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index 2c53003112a..7a1f24fdef6 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -81,16 +81,16 @@ LoadUtils::loadUDAT(const AttributeVector& attr) #define INSTANTIATE_ARRAY(ValueType, Saver) \ -template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) +template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_WSET(ValueType, Saver) \ -template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<WeightedValue<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) +template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<WeightedValue<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE(ValueType, Saver) \ -template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<load_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) +template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE_ARRAY_WSET(ValueType, Saver) \ INSTANTIATE_SINGLE(ValueType, Saver); \ -INSTANTIATE_ARRAY(load_utils::NonAtomicValue_t<ValueType>, Saver); \ -INSTANTIATE_WSET(load_utils::NonAtomicValue_t<ValueType>, Saver) +INSTANTIATE_ARRAY(ValueType, Saver); \ +INSTANTIATE_WSET(ValueType, Saver) #define INSTANTIATE_ENUM(Saver) \ INSTANTIATE_SINGLE_ARRAY_WSET(AtomicEntryRef, Saver) diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index fe41811dcfa..ec4f4862ae1 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -2,6 +2,7 @@ #pragma once +#include "atomic_utils.h" #include "attributevector.h" #include "readerbase.h" #include <vespa/vespalib/util/arrayref.h> @@ -15,30 +16,6 @@ class EntryRef; namespace search::attribute { -namespace load_utils { - -/* - * Helper class to map from atomic value to non-atomic value, e.g. - * from AtomicEntryRef to EntryRef. - */ -template <typename MaybeAtomicValue> -class NonAtomicValue { -public: - using type = MaybeAtomicValue; -}; - -template <> -class NonAtomicValue<vespalib::datastore::AtomicEntryRef> -{ -public: - using type = vespalib::datastore::EntryRef; -}; - -template <class MaybeAtomicValue> -using NonAtomicValue_t = typename NonAtomicValue<MaybeAtomicValue>::type; - -} - /** * Helper functions used to open / load attribute vector data files from disk. */ @@ -69,7 +46,7 @@ template <class MvMapping, class Saver> uint32_t loadFromEnumeratedMultiValue(MvMapping &mapping, ReaderBase &attrReader, - vespalib::ConstArrayRef<typename MvMapping::MultiValueType::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename MvMapping::MultiValueType::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) __attribute((noinline)); @@ -82,7 +59,7 @@ void loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, - vespalib::ConstArrayRef<load_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) __attribute((noinline)); diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 92cbc72ae2c..76b19f40ced 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -11,12 +11,14 @@ template <class MvMapping, class Saver> uint32_t loadFromEnumeratedMultiValue(MvMapping & mapping, ReaderBase & attrReader, - vespalib::ConstArrayRef<typename MvMapping::MultiValueType::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename MvMapping::MultiValueType::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) { mapping.prepareLoadFromMultiValue(); using MultiValueType = typename MvMapping::MultiValueType; + using ValueType = typename MultiValueType::ValueType; + using NonAtomicValueType = atomic_utils::NonAtomicValue_t<ValueType>; std::vector<MultiValueType> indices; uint32_t numDocs = attrReader.getNumIdx() - 1; uint64_t numValues = attrReader.getEnumCount(); @@ -35,7 +37,11 @@ loadFromEnumeratedMultiValue(MvMapping & mapping, enumValue = enum_value_remapping[enumValue]; } int32_t weight = MultiValueType::_hasWeight ? attrReader.getNextWeight() : 1; - indices.emplace_back(enumValueToValueMap[enumValue], weight); + if constexpr (std::is_same_v<ValueType, NonAtomicValueType>) { + indices.emplace_back(enumValueToValueMap[enumValue], weight); + } else { + indices.emplace_back(ValueType(enumValueToValueMap[enumValue]), weight); + } saver.save(enumValue, doc, weight); } if (maxValueCount < indices.size()) { @@ -54,12 +60,12 @@ void loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, - vespalib::ConstArrayRef<load_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) { using ValueType = typename Vector::ValueType; - using NonAtomicValueType = load_utils::NonAtomicValue_t<ValueType>; + using NonAtomicValueType = atomic_utils::NonAtomicValue_t<ValueType>; uint32_t numDocs = attrReader.getEnumCount(); genHolder.clearHoldLists(); vector.reset(); diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp index 779e020cdd0..b514275f75d 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp @@ -3,8 +3,7 @@ #include "loadedenumvalue.h" #include <vespa/searchlib/common/sort.h> -namespace search { -namespace attribute { +namespace search::attribute { void sortLoadedByEnum(LoadedEnumAttributeVector &loaded) @@ -17,6 +16,5 @@ sortLoadedByEnum(LoadedEnumAttributeVector &loaded) &loaded[0], loaded.size(), 16); } -} // namespace attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h index 96bdc147559..bbd811095c6 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h @@ -3,7 +3,6 @@ #pragma once #include "enum_store_types.h" -#include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/arrayref.h> #include <cassert> #include <limits> @@ -67,7 +66,7 @@ public: int32_t getWeight() const { return _weight; } }; -typedef vespalib::Array<LoadedEnumAttribute> LoadedEnumAttributeVector; +using LoadedEnumAttributeVector = std::vector<LoadedEnumAttribute, vespalib::allocator_large<LoadedEnumAttribute>>; /** diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp index 161e2883cbc..ffd380c174b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp @@ -5,6 +5,7 @@ #include "multi_value_mapping.h" #include "multi_value_mapping.hpp" #include "multivalue.h" +#include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/array.hpp> @@ -13,8 +14,8 @@ using search::multivalue::WeightedValue; namespace search::attribute { -template class MultiValueMapping<Value<IEnumStore::Index>>; -template class MultiValueMapping<WeightedValue<IEnumStore::Index>>; +template class MultiValueMapping<Value<vespalib::datastore::AtomicEntryRef>>; +template class MultiValueMapping<WeightedValue<vespalib::datastore::AtomicEntryRef>>; template class MultiValueMapping<Value<int8_t>>; template class MultiValueMapping<WeightedValue<int8_t>>; template class MultiValueMapping<Value<int16_t>>; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index f5f2950a59c..71f1191e507 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -18,6 +18,7 @@ public: using MultiValueType = EntryT; using RefType = RefT; private: + using ArrayRef = vespalib::ArrayRef<EntryT>; using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; using generation_t = vespalib::GenerationHandler::generation_t; using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; @@ -30,13 +31,13 @@ public: const vespalib::GrowStrategy &gs, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~MultiValueMapping() override; - ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId]); } + ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId].load_acquire()); } ConstArrayRef getDataForIdx(EntryRef idx) const { return _store.get(idx); } void set(uint32_t docId, ConstArrayRef values); - // replace is generally unsafe and should only be used when + // get_writable is generally unsafe and should only be used when // compacting enum store (replacing old enum index with updated enum index) - void replace(uint32_t docId, ConstArrayRef values); + ArrayRef get_writable(uint32_t docId) { return _store.get_writable(_indices[docId].load_relaxed()); } // Pass on hold list management to underlying store void transferHoldLists(generation_t generation) { _store.transferHoldLists(generation); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 16b29bf33cd..87ffaa75ace 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -32,33 +32,20 @@ void MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values) { _indices.ensure_size(docId + 1); - EntryRef oldRef(_indices[docId]); + EntryRef oldRef(_indices[docId].load_relaxed()); ConstArrayRef oldValues = _store.get(oldRef); - _indices[docId] = _store.add(values); + _indices[docId].store_release(_store.add(values)); updateValueCount(oldValues.size(), values.size()); _store.remove(oldRef); } template <typename EntryT, typename RefT> void -MultiValueMapping<EntryT,RefT>::replace(uint32_t docId, ConstArrayRef values) -{ - auto oldValues = _store.get_writable(_indices[docId]); - assert(oldValues.size() == values.size()); - EntryT *dst = &oldValues[0]; - for (auto &src : values) { - *dst = src; - ++dst; - } -} - -template <typename EntryT, typename RefT> -void MultiValueMapping<EntryT,RefT>::compactWorst(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { vespalib::datastore::ICompactionContext::UP compactionContext(_store.compactWorst(compaction_spec, compaction_strategy)); if (compactionContext) { - compactionContext->compact(vespalib::ArrayRef<EntryRef>(&_indices[0], _indices.size())); + compactionContext->compact(vespalib::ArrayRef<AtomicEntryRef>(&_indices[0], _indices.size())); } } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index 7ad61ccedc5..11ea58bc124 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -3,6 +3,7 @@ #include "multi_value_mapping_base.h" #include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/compaction_strategy.h> +#include <vespa/vespalib/util/array.hpp> #include <cassert> namespace search::attribute { @@ -24,14 +25,19 @@ MultiValueMappingBase::~MultiValueMappingBase() = default; MultiValueMappingBase::RefCopyVector MultiValueMappingBase::getRefCopy(uint32_t size) const { assert(size <= _indices.size()); - return RefCopyVector(&_indices[0], &_indices[0] + size); + RefCopyVector result; + result.reserve(size); + for (uint32_t lid = 0; lid < size; ++lid) { + result.push_back(_indices[lid].load_relaxed()); + } + return result; } void MultiValueMappingBase::addDoc(uint32_t & docId) { uint32_t retval = _indices.size(); - _indices.push_back(EntryRef()); + _indices.push_back(AtomicEntryRef()); docId = retval; } @@ -54,7 +60,7 @@ MultiValueMappingBase::clearDocs(uint32_t lidLow, uint32_t lidLimit, std::functi assert(lidLow <= lidLimit); assert(lidLimit <= _indices.size()); for (uint32_t lid = lidLow; lid < lidLimit; ++lid) { - if (_indices[lid].valid()) { + if (_indices[lid].load_relaxed().valid()) { clearDoc(lid); } } @@ -90,3 +96,9 @@ MultiValueMappingBase::considerCompact(const CompactionStrategy &compactionStrat } } + +namespace vespalib { + +template class Array<datastore::AtomicEntryRef>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h index 2b2b4d5f8a3..e4588c8e743 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.h @@ -2,8 +2,8 @@ #pragma once +#include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/datastore/compaction_spec.h> -#include <vespa/vespalib/datastore/entryref.h> #include <vespa/vespalib/util/address_space.h> #include <vespa/vespalib/util/rcuvector.h> #include <functional> @@ -23,8 +23,9 @@ class MultiValueMappingBase public: using CompactionSpec = vespalib::datastore::CompactionSpec; using CompactionStrategy = vespalib::datastore::CompactionStrategy; + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using EntryRef = vespalib::datastore::EntryRef; - using RefVector = vespalib::RcuVectorBase<EntryRef>; + using RefVector = vespalib::RcuVectorBase<AtomicEntryRef>; protected: std::shared_ptr<vespalib::alloc::MemoryAllocator> _memory_allocator; diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp index 8790bdd9885..938981689da 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp @@ -17,15 +17,13 @@ namespace search::multienumattribute { using EnumIndex = IEnumStore::Index; using EnumIndexRemapper = IEnumStore::EnumIndexRemapper; -using Value = multivalue::Value<EnumIndex>; -using WeightedValue = multivalue::WeightedValue<EnumIndex>; +using Value = multivalue::Value<vespalib::datastore::AtomicEntryRef>; +using WeightedValue = multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef>; template <typename WeightedIndex> void remap_enum_store_refs(const EnumIndexRemapper& remapper, AttributeVector& v, attribute::MultiValueMapping<WeightedIndex>& multi_value_mapping) { - using WeightedIndexVector = std::vector<WeightedIndex>; - // update multi_value_mapping with new EnumIndex values after enum store has been compacted. v.logEnumStoreEvent("compactfixup", "drain"); { @@ -33,17 +31,14 @@ remap_enum_store_refs(const EnumIndexRemapper& remapper, AttributeVector& v, att auto& filter = remapper.get_entry_ref_filter(); v.logEnumStoreEvent("compactfixup", "start"); for (uint32_t doc = 0; doc < v.getNumDocs(); ++doc) { - vespalib::ConstArrayRef<WeightedIndex> indicesRef(multi_value_mapping.get(doc)); - WeightedIndexVector indices(indicesRef.cbegin(), indicesRef.cend()); - for (uint32_t i = 0; i < indices.size(); ++i) { - EnumIndex ref = indices[i].value(); + vespalib::ArrayRef<WeightedIndex> indices(multi_value_mapping.get_writable(doc)); + for (auto& entry : indices) { + EnumIndex ref = entry.value_ref().load_relaxed(); if (ref.valid() && filter.has(ref)) { ref = remapper.remap(ref); + entry.value_ref().store_release(ref); } - indices[i] = WeightedIndex(ref, indices[i].weight()); } - std::atomic_thread_fence(std::memory_order_release); - multi_value_mapping.replace(doc, indices); } } v.logEnumStoreEvent("compactfixup", "complete"); diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.h b/searchlib/src/vespa/searchlib/attribute/multienumattribute.h index de4e3dcdace..fb3e66ac60a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.h @@ -2,7 +2,6 @@ #pragma once -#include "enum_store_loaders.h" #include "i_enum_store.h" #include "loadedenumvalue.h" #include "multivalue.h" @@ -14,7 +13,7 @@ namespace search { class IWeightedIndexVector { public: virtual ~IWeightedIndexVector() = default; - using WeightedIndex = multivalue::WeightedValue<IEnumStore::Index>; + using WeightedIndex = multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef>; /** * Provides a reference to the underlying enum/weight pairs. * This method should only be invoked if @ref getCollectionType(docId) returns CollectionType::WEIGHTED_SET. @@ -40,6 +39,7 @@ class MultiValueEnumAttribute : public MultiValueAttribute<B, M>, public IWeightedIndexVector { protected: + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using Change = typename B::BaseClass::Change; using DocId = typename B::BaseClass::DocId; using EnumHandle = typename B::BaseClass::EnumHandle; @@ -92,7 +92,7 @@ public: if (indices.size() == 0) { return std::numeric_limits<uint32_t>::max(); } else { - return indices[0].value().ref(); + return indices[0].value_ref().load_acquire().ref(); } } @@ -100,7 +100,7 @@ public: WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { - e[i] = indices[i].value().ref(); + e[i] = indices[i].value_ref().load_acquire().ref(); } return valueCount; } @@ -108,7 +108,7 @@ public: WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { - e[i] = WeightedEnum(indices[i].value().ref(), indices[i].weight()); + e[i] = WeightedEnum(indices[i].value_ref().load_acquire().ref(), indices[i].weight()); } return valueCount; } diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 7df30b895d2..bb1e4712c2d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -60,10 +60,10 @@ MultiValueEnumAttribute<B, M>::applyValueChanges(const DocIndices& docIndices, E uint32_t valueCount = oldIndices.size(); this->_mvMapping.set(doc_values.first, doc_values.second); for (uint32_t i = 0; i < doc_values.second.size(); ++i) { - updater.inc_ref_count(doc_values.second[i]); + updater.inc_ref_count(doc_values.second[i].value_ref().load_relaxed()); } for (uint32_t i = 0; i < valueCount; ++i) { - updater.dec_ref_count(oldIndices[i]); + updater.dec_ref_count(oldIndices[i].value_ref().load_relaxed()); } } } @@ -80,7 +80,7 @@ MultiValueEnumAttribute<B, M>::fillValues(LoadedVector & loaded) this->_mvMapping.prepareLoadFromMultiValue(); for (DocId doc = 0; doc < numDocs; ++doc) { for(const auto* v = & loaded.read();(count < numValues) && (v->_docId == doc); count++, loaded.next(), v = & loaded.read()) { - indices.push_back(WeightedIndex(v->getEidx(), v->getWeight())); + indices.push_back(WeightedIndex(AtomicEntryRef(v->getEidx()), v->getWeight())); } this->checkSetMaxValueCount(indices.size()); this->_mvMapping.set(doc, indices); diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp index c058653bdcd..41a687c6fad 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp @@ -68,7 +68,7 @@ public: if (_indexes.size() >= _indexes.capacity()) { flush(); } - _indexes.push_back(valueRef.value()); + _indexes.push_back(valueRef.value_ref().load_acquire()); } } }; @@ -134,8 +134,8 @@ onSave(IAttributeSaveTarget &saveTarget) return !compaction_broke_save; } -using EnumIdxArray = multivalue::Value<IEnumStore::Index>; -using EnumIdxWset = multivalue::WeightedValue<IEnumStore::Index>; +using EnumIdxArray = multivalue::Value<vespalib::datastore::AtomicEntryRef>; +using EnumIdxWset = multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef>; template class MultiValueEnumAttributeSaver<EnumIdxArray>; template class MultiValueEnumAttributeSaver<EnumIdxWset>; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index 85d01aaef78..1c0124f6a1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -64,7 +64,7 @@ protected: { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { - T v = _toBeSearched._enumStore.get_value(indices[i].value()); + T v = _toBeSearched._enumStore.get_value(indices[i].value_ref().load_acquire()); if (this->match(v)) { weight = indices[i].weight(); return i; @@ -78,7 +78,7 @@ protected: { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { - T v = _toBeSearched._enumStore.get_value(indices[i].value()); + T v = _toBeSearched._enumStore.get_value(indices[i].value_ref().load_acquire()); if (this->match(v)) { return i; } @@ -118,7 +118,7 @@ protected: { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { - T v = _toBeSearched._enumStore.get_value(indices[i].value()); + T v = _toBeSearched._enumStore.get_value(indices[i].value_ref().load_acquire()); if (this->match(v)) { weight = 1; return i; @@ -134,7 +134,7 @@ protected: { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { - T v = _toBeSearched._enumStore.get_value(indices[i].value()); + T v = _toBeSearched._enumStore.get_value(indices[i].value_ref().load_acquire()); if (this->match(v)) { return i; } @@ -166,7 +166,7 @@ public: if (indices.size() == 0) { return T(); } else { - return this->_enumStore.get_value(indices[0].value()); + return this->_enumStore.get_value(indices[0].value_ref().load_acquire()); } } largeint_t getInt(DocId doc) const override { @@ -181,7 +181,7 @@ public: WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); uint32_t valueCount = indices.size(); for(uint32_t i = 0, m = std::min(sz, valueCount); i < m; i++) { - buffer[i] = static_cast<BufferType>(this->_enumStore.get_value(indices[i].value())); + buffer[i] = static_cast<BufferType>(this->_enumStore.get_value(indices[i].value_ref().load_acquire())); } return valueCount; } @@ -200,7 +200,7 @@ public: WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { - buffer[i] = WeightedType(static_cast<ValueType>(this->_enumStore.get_value(indices[i].value())), indices[i].weight()); + buffer[i] = WeightedType(static_cast<ValueType>(this->_enumStore.get_value(indices[i].value_ref().load_acquire())), indices[i].weight()); } return valueCount; } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp index c89d154949e..376078763f1 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp @@ -5,8 +5,8 @@ namespace search { -template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<IEnumStore::Index> >; -template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<IEnumStore::Index> >; +template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<vespalib::datastore::AtomicEntryRef> >; +template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef> >; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 5d2a5237004..415bfa25b16 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -2,11 +2,11 @@ #pragma once -#include <vespa/searchlib/attribute/stringbase.h> -#include <vespa/searchlib/attribute/enumattribute.h> -#include <vespa/searchlib/attribute/enumstore.h> -#include <vespa/searchlib/attribute/multienumattribute.h> -#include <vespa/searchlib/attribute/multi_value_mapping.h> +#include "stringbase.h" +#include "enumattribute.h" +#include "enumstore.h" +#include "multienumattribute.h" +#include "multi_value_mapping.h" #include "enumhintsearchcontext.h" #include "multivalue.h" @@ -61,7 +61,7 @@ public: if (indices.size() == 0) { return NULL; } else { - return this->_enumStore.get_value(indices[0].value()); + return this->_enumStore.get_value(indices[0].value_ref().load_acquire()); } } @@ -77,7 +77,7 @@ public: WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); uint32_t valueCount = indices.size(); for(uint32_t i = 0, m = std::min(sz, valueCount); i < m; i++) { - buffer[i] = this->_enumStore.get_value(indices[i].value()); + buffer[i] = this->_enumStore.get_value(indices[i].value_ref().load_acquire()); } return valueCount; } @@ -94,7 +94,7 @@ public: WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { - buffer[i] = WeightedType(this->_enumStore.get_value(indices[i].value()), indices[i].weight()); + buffer[i] = WeightedType(this->_enumStore.get_value(indices[i].value_ref().load_acquire()), indices[i].weight()); } return valueCount; } @@ -162,7 +162,7 @@ public: }; -using ArrayStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<IEnumStore::Index> >; -using WeightedSetStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<IEnumStore::Index> >; +using ArrayStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<vespalib::datastore::AtomicEntryRef> >; +using WeightedSetStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef> >; } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index c8330225fb9..ca440c2a249 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -97,7 +97,7 @@ MultiValueStringAttributeT<B, M>::StringImplSearchContext::onFind(DocId doc, int const auto& attr = static_cast<const MultiValueStringAttributeT<B, M>&>(attribute()); WeightedIndexArrayRef indices(attr._mvMapping.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { - if (isMatch(attr._enumStore.get_value(indices[i].value()))) { + if (isMatch(attr._enumStore.get_value(indices[i].value_ref().load_acquire()))) { return i; } } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index 9fe1522709b..e2993835d83 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -106,8 +106,8 @@ public: } }; -using ArrayStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<IEnumStore::Index> >; -using WeightedSetStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<IEnumStore::Index> >; +using ArrayStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<vespalib::datastore::AtomicEntryRef> >; +using WeightedSetStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<vespalib::datastore::AtomicEntryRef> >; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalue.h b/searchlib/src/vespa/searchlib/attribute/multivalue.h index eb2c27cf670..780e8627b91 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalue.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalue.h @@ -14,6 +14,8 @@ public: Value(T v) noexcept : _v(v) { } Value(T v, int32_t w) noexcept : _v(v) { (void) w; } T value() const { return _v; } + const T& value_ref() const { return _v; } + T& value_ref() { return _v; } operator T () const { return _v; } operator T & () { return _v; } int32_t weight() const { return 1; } @@ -36,6 +38,8 @@ public: WeightedValue() noexcept : _v(), _w(1) { } WeightedValue(T v, int32_t w) noexcept : _v(v), _w(w) { } T value() const { return _v; } + const T& value_ref() const { return _v; } + T& value_ref() { return _v; } operator T () const { return _v; } operator T & () { return _v; } int32_t weight() const { return _w; } diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index a2d4d598c47..64e49597aff 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -4,6 +4,7 @@ #include "multi_value_mapping.h" #include "attributevector.h" +#include "atomic_utils.h" namespace search { @@ -28,6 +29,7 @@ protected: using MultiValueArrayRef = vespalib::ConstArrayRef<MultiValueType>; typedef typename ValueVector::iterator ValueVectorIterator; typedef std::vector<std::pair<DocId, ValueVector> > DocumentValues; + using NonAtomicValueType = attribute::atomic_utils::NonAtomicValue_t<ValueType>; MultiValueMapping _mvMapping; @@ -39,7 +41,7 @@ protected: */ void applyAttributeChanges(DocumentValues & docValues); - virtual bool extractChangeData(const Change & c, ValueType & data) = 0; + virtual bool extractChangeData(const Change & c, NonAtomicValueType & data) = 0; /** * Called when a new document has been added. diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 326dbd4a380..e009a3b69f6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -95,7 +95,7 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_array(DocumentValues& docV } MultiValueArrayRef old_values(_mvMapping.get(doc)); ValueVector new_values(old_values.cbegin(), old_values.cend()); - vespalib::hash_map<ValueType, size_t, typename HashFn<ValueType>::type> tombstones; + vespalib::hash_map<NonAtomicValueType, size_t, typename HashFn<NonAtomicValueType>::type> tombstones; // iterate through all changes for this document for (; (current != end) && (current->_doc == doc); ++current) { @@ -104,13 +104,17 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_array(DocumentValues& docV tombstones.clear(); continue; } - ValueType data; + NonAtomicValueType data; bool hasData = extractChangeData(*current, data); if (!hasData) { continue; } if (current->_type == ChangeBase::APPEND) { - new_values.emplace_back(data, current->_weight); + if constexpr (std::is_same_v<ValueType, NonAtomicValueType>) { + new_values.emplace_back(data, current->_weight); + } else { + new_values.emplace_back(ValueType(data), current->_weight); + } } else if (current->_type == ChangeBase::REMOVE) { // Defer all removals to the very end by tracking when, during value vector build time, // a removal was encountered for a particular value. All values < this index will be ignored. @@ -121,10 +125,19 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_array(DocumentValues& docV if (!tombstones.empty()) { ValueVector culled; culled.reserve(new_values.size()); - for (size_t i = 0; i < new_values.size(); ++i) { - auto iter = tombstones.find(new_values[i].value()); - if (iter == tombstones.end() || (iter->second <= i)) { - culled.emplace_back(new_values[i]); + if constexpr (std::is_same_v<ValueType, NonAtomicValueType>) { + for (size_t i = 0; i < new_values.size(); ++i) { + auto iter = tombstones.find(new_values[i].value()); + if (iter == tombstones.end() || (iter->second <= i)) { + culled.emplace_back(new_values[i]); + } + } + } else { + for (size_t i = 0; i < new_values.size(); ++i) { + auto iter = tombstones.find(new_values[i].value_ref().load_relaxed()); + if (iter == tombstones.end() || (iter->second <= i)) { + culled.emplace_back(new_values[i]); + } } } culled.swap(new_values); @@ -156,10 +169,14 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docVa current = last_clear_doc; } MultiValueArrayRef old_values(_mvMapping.get(doc)); - vespalib::hash_map<ValueType, int32_t, typename HashFn<ValueType>::type> wset_inserted; + vespalib::hash_map<NonAtomicValueType, int32_t, typename HashFn<NonAtomicValueType>::type> wset_inserted; wset_inserted.resize((old_values.size() + max_elems_inserted) * 2); for (const auto& e : old_values) { - wset_inserted[e.value()] = e.weight(); + if constexpr (std::is_same_v<ValueType, NonAtomicValueType>) { + wset_inserted[e.value()] = e.weight(); + } else { + wset_inserted[e.value_ref().load_relaxed()] = e.weight(); + } } // iterate through all changes for this document for (; (current != end) && (current->_doc == doc); ++current) { @@ -167,7 +184,7 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docVa wset_inserted.clear(); continue; } - ValueType data; + NonAtomicValueType data; bool hasData = extractChangeData(*current, data); if (!hasData) { continue; @@ -193,7 +210,11 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docVa } std::vector<MultiValueType> new_values; new_values.reserve(wset_inserted.size()); - wset_inserted.for_each([&new_values](const auto& e){ new_values.emplace_back(e.first, e.second); }); + if constexpr (std::is_same_v<ValueType, NonAtomicValueType>) { + wset_inserted.for_each([&new_values](const auto& e){ new_values.emplace_back(e.first, e.second); }); + } else { + wset_inserted.for_each([&new_values](const auto& e){ new_values.emplace_back(ValueType(e.first), e.second); }); + } this->checkSetMaxValueCount(new_values.size()); docValues.emplace_back(doc, std::move(new_values)); diff --git a/searchlib/src/vespa/searchlib/attribute/numericbase.h b/searchlib/src/vespa/searchlib/attribute/numericbase.h index bd3a68fdc60..bd6ffda1b19 100644 --- a/searchlib/src/vespa/searchlib/attribute/numericbase.h +++ b/searchlib/src/vespa/searchlib/attribute/numericbase.h @@ -5,7 +5,6 @@ #include "attributevector.h" #include "i_enum_store.h" #include "loadedenumvalue.h" -#include "enum_store_loaders.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index 9081d33f91e..a06ee9dab2b 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -8,10 +8,20 @@ #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> +using vespalib::datastore::AtomicEntryRef; + namespace search { namespace { +template <typename WeightedIndex> +struct CompareValue { + bool operator()(const WeightedIndex& lhs, const WeightedIndex& rhs) const + { + return lhs.value_ref().load_relaxed() < rhs.value_ref().load_relaxed(); + }; +}; + void removeDupAdditions(PostingChange<AttributePosting>::A &additions) { @@ -133,7 +143,7 @@ template <typename WeightedIndex> class ActualChangeComputer { public: using EnumIndex = IEnumStore::Index; - using AlwaysWeightedIndexVector = std::vector<multivalue::WeightedValue<EnumIndex>>; + using AlwaysWeightedIndexVector = std::vector<multivalue::WeightedValue<AtomicEntryRef>>; using WeightedIndexVector = std::vector<WeightedIndex>; void compute(const WeightedIndex * entriesNew, size_t szNew, const WeightedIndex * entriesOld, size_t szOld, @@ -168,7 +178,7 @@ private: { const WeightedIndex *srce = src + sz; for (const WeightedIndex *i = src; i < srce; ++i) { - dst.emplace_back(mapEnumIndex(i->value()), i->weight()); + dst.emplace_back(AtomicEntryRef(mapEnumIndex(i->value_ref().load_relaxed())), i->weight()); } } @@ -181,7 +191,7 @@ private: } else { copyFast(dst, src, sz); } - std::sort(dst.begin(), dst.end()); + std::sort(dst.begin(), dst.end(), CompareValue<WeightedIndex>()); } }; @@ -189,21 +199,21 @@ template <typename WeightedIndex> class MergeDupIterator { using InnerIter = typename std::vector<WeightedIndex>::const_iterator; using EnumIndex = IEnumStore::Index; - using Entry = multivalue::WeightedValue<EnumIndex>; + using Entry = multivalue::WeightedValue<AtomicEntryRef>; InnerIter _cur; InnerIter _end; Entry _entry; bool _valid; void merge() { - EnumIndex idx = _cur->value(); + EnumIndex idx = _cur->value_ref().load_relaxed(); int32_t weight = _cur->weight(); ++_cur; - while (_cur != _end && _cur->value() == idx) { + while (_cur != _end && _cur->value_ref().load_relaxed() == idx) { // sum weights together. Overflow is not handled. weight += _cur->weight(); ++_cur; } - _entry = Entry(idx, weight); + _entry = Entry(AtomicEntryRef(idx), weight); } public: MergeDupIterator(const std::vector<WeightedIndex> &vec) @@ -219,7 +229,7 @@ public: bool valid() const { return _valid; } const Entry &entry() const { return _entry; } - EnumIndex value() const { return _entry.value(); } + EnumIndex value() const { return _entry.value_ref().load_relaxed(); } int32_t weight() const { return _entry.weight(); } void next() { if (_cur != _end) { @@ -301,13 +311,13 @@ compute(const MultivalueMapping & mvm, const DocIndices & docIndices, actualChange.compute(&docIndex.second[0], docIndex.second.size(), &oldIndices[0], oldIndices.size(), added, changed, removed); for (const auto & wi : added) { - changePost[EnumPostingPair(wi.value(), &compare)].add(docIndex.first, wi.weight()); + changePost[EnumPostingPair(wi.value_ref().load_relaxed(), &compare)].add(docIndex.first, wi.weight()); } for (const auto & wi : removed) { - changePost[EnumPostingPair(wi.value(), &compare)].remove(docIndex.first); + changePost[EnumPostingPair(wi.value_ref().load_relaxed(), &compare)].remove(docIndex.first); } for (const auto & wi : changed) { - changePost[EnumPostingPair(wi.value(), &compare)].remove(docIndex.first).add(docIndex.first, wi.weight()); + changePost[EnumPostingPair(wi.value_ref().load_relaxed(), &compare)].remove(docIndex.first).add(docIndex.first, wi.weight()); } } return changePost; @@ -319,9 +329,8 @@ template class PostingChange<AttributeWeightPosting>; typedef PostingChange<vespalib::btree::BTreeKeyData<unsigned int, int> > WeightedPostingChange; typedef std::map<EnumPostingPair, WeightedPostingChange> WeightedPostingChangeMap; -typedef IEnumStore::Index EnumIndex; -typedef multivalue::WeightedValue<EnumIndex> WeightedIndex; -typedef multivalue::Value<EnumIndex> ValueIndex; +typedef multivalue::WeightedValue<AtomicEntryRef> WeightedIndex; +typedef multivalue::Value<AtomicEntryRef> ValueIndex; using WeightedMultiValueMapping = attribute::MultiValueMapping<WeightedIndex>; using ValueMultiValueMapping = attribute::MultiValueMapping<ValueIndex>; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index c28131159e7..742b67f20ae 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -3,6 +3,7 @@ #include "postinglistattribute.h" #include "loadednumericvalue.h" #include "enumcomparator.h" +#include "enum_store_loaders.h" #include <vespa/vespalib/util/array.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index bb454a8b0d4..608f37c0cfa 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -7,6 +7,7 @@ #include "ipostinglistattributebase.h" #include "singleenumattributesaver.h" #include "load_utils.h" +#include "enum_store_loaders.h" #include <vespa/vespalib/datastore/unique_store_remapper.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/stringattribute.h b/searchlib/src/vespa/searchlib/attribute/stringattribute.h index bd0ca998a1a..70508f14168 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/stringattribute.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/searchlib/attribute/stringbase.h> -#include <vespa/searchlib/attribute/enumstore.h> +#include "stringbase.h" +#include "enumstore.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 57882ddf1ae..6062c4f2096 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -4,6 +4,7 @@ #include "load_utils.h" #include "readerbase.h" #include "stringbase.h" +#include "enum_store_loaders.h" #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/searchlib/query/query_term_ucs4.h> #include <vespa/searchlib/util/fileutil.hpp> diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index 3d3a2bc0b3d..495427d3e45 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -3,11 +3,10 @@ #pragma once #include "no_loaded_vector.h" -#include "enum_store_loaders.h" -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/changevector.h> -#include <vespa/searchlib/attribute/i_enum_store.h> -#include <vespa/searchlib/attribute/loadedenumvalue.h> +#include "attributevector.h" +#include "changevector.h" +#include "i_enum_store.h" +#include "loadedenumvalue.h" #include <vespa/vespalib/regex/regex.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/vespalib/text/utf8.h> @@ -158,7 +157,7 @@ protected: template<typename WeightedT, typename Accessor, typename Collector> int32_t findNextMatch(vespalib::ConstArrayRef<WeightedT> w, int32_t elemId, const Accessor & ac, Collector & collector) const { for (uint32_t i(elemId); i < w.size(); i++) { - if (isMatch(ac.get(w[i].value()))) { + if (isMatch(ac.get(w[i].value_ref().load_acquire()))) { collector.addWeight(w[i].weight()); return i; } diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index c1d447047d1..02f51fe8758 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -6,7 +6,7 @@ #include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/generationholder.h> -#include <vespa/fastos/dynamiclibrary.h> +#include <vespa/fastos/types.h> namespace vespalib { class nbostream; diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.h b/searchlib/src/vespa/searchlib/common/bitvectorcache.h index 33610afc108..5f8edf29854 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectorcache.h +++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.h @@ -4,7 +4,6 @@ #include "condensedbitvectors.h" #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/fastos/dynamiclibrary.h> #include <mutex> namespace search { diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h index e91e06bdf7a..07467e28229 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h @@ -5,7 +5,6 @@ #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/queryeval/iterators.h> -#include <vespa/fastos/dynamiclibrary.h> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index 60c788cf9c7..e57fba712fd 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -8,6 +8,7 @@ #include "randread.h" #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/memoryusage.h> diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index 05eb7880ae8..1b5f9dc1204 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -66,7 +66,7 @@ private: uint32_t _chunkId; uint32_t _lid; }; - using IndexVector = vespalib::Array<Index>; + using IndexVector = std::vector<Index, vespalib::allocator_large<Index>>; uint64_t _chunkSerial; Chunk::UP _current; std::map<uint64_t, IndexVector> _where; diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 4a32f1b7078..59ea82c83bb 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -193,7 +193,7 @@ DotProductExecutorByEnum::execute(uint32_t docId) { const IWeightedIndexVector::WeightedIndex *values(nullptr); uint32_t sz = _attribute->getEnumHandles(docId, values); for (size_t i = 0; i < sz; ++i) { - auto itr = _queryVector.getDimMap().find(values[i].value().ref()); + auto itr = _queryVector.getDimMap().find(values[i].value_ref().load_relaxed().ref()); if (itr != _end) { val += values[i].weight() * itr->second; } @@ -213,7 +213,7 @@ public: const IWeightedIndexVector::WeightedIndex *values(nullptr); uint32_t sz = _attribute->getEnumHandles(docId, values); for (size_t i = 0; i < sz; ++i) { - if (values[i].value().ref() == _key) { + if (values[i].value_ref().load_relaxed().ref() == _key) { outputs().set_number(0, values[i].weight()*_value); return; } diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h index d9375f12d54..1d5ac98a74d 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h @@ -4,7 +4,6 @@ #include "nativerankfeature.h" #include "queryterm.h" -#include <vespa/fastos/dynamiclibrary.h> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp b/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp index 5092842e4b3..c0d7beac260 100644 --- a/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp +++ b/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp @@ -17,10 +17,11 @@ FeatureOverrider::handle_bind_outputs(vespalib::ArrayRef<NumberOrObject> outputs _executor.bind_outputs(outputs); } -FeatureOverrider::FeatureOverrider(FeatureExecutor &executor, uint32_t outputIdx, feature_t value) +FeatureOverrider::FeatureOverrider(FeatureExecutor &executor, uint32_t outputIdx, feature_t number, Value::UP object) : _executor(executor), _outputIdx(outputIdx), - _value(value) + _number(number), + _object(std::move(object)) { } @@ -35,7 +36,11 @@ FeatureOverrider::execute(uint32_t docId) { _executor.lazy_execute(docId); if (_outputIdx < outputs().size()) { - outputs().set_number(_outputIdx, _value); + if (_object) { + outputs().set_object(_outputIdx, *_object); + } else { + outputs().set_number(_outputIdx, _number); + } } } diff --git a/searchlib/src/vespa/searchlib/fef/featureoverrider.h b/searchlib/src/vespa/searchlib/fef/featureoverrider.h index 05c651932fe..1ce18903e4e 100644 --- a/searchlib/src/vespa/searchlib/fef/featureoverrider.h +++ b/searchlib/src/vespa/searchlib/fef/featureoverrider.h @@ -17,31 +17,25 @@ namespace fef { class FeatureOverrider : public FeatureExecutor { private: + using Value = vespalib::eval::Value; + FeatureOverrider(const FeatureOverrider &); FeatureOverrider &operator=(const FeatureOverrider &); FeatureExecutor & _executor; uint32_t _outputIdx; - feature_t _value; + feature_t _number; + Value::UP _object; virtual void handle_bind_match_data(const MatchData &md) override; virtual void handle_bind_inputs(vespalib::ConstArrayRef<LazyValue> inputs) override; virtual void handle_bind_outputs(vespalib::ArrayRef<NumberOrObject> outputs) override; public: - /** - * Create a feature overrider that will override the given output - * with the given feature value. - * - * @param executor the feature executor for which we should override an output - * @param outputIdx which output to override - * @param value what value to override with - **/ - FeatureOverrider(FeatureExecutor &executor, uint32_t outputIdx, feature_t value); + FeatureOverrider(FeatureExecutor &executor, uint32_t outputIdx, feature_t number, Value::UP object); bool isPure() override; void execute(uint32_t docId) override; }; } // namespace fef } // namespace search - diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index 83cb9237975..c8c2d59e872 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -3,8 +3,11 @@ #include "rank_program.h" #include "featureoverrider.h" #include <vespa/vespalib/locale/c.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/issue.h> #include <algorithm> #include <cassert> @@ -12,6 +15,10 @@ LOG_SETUP(".fef.rankprogram"); using vespalib::Stash; +using vespalib::Issue; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::eval::FastValueBuilderFactory; namespace search::fef { @@ -22,10 +29,14 @@ namespace { struct Override { BlueprintResolver::FeatureRef ref; - feature_t value; + feature_t number; + Value::UP object; Override(const BlueprintResolver::FeatureRef &r, feature_t v) noexcept - : ref(r), value(v) {} + : ref(r), number(v), object() {} + + Override(const BlueprintResolver::FeatureRef &r, Value::UP v) noexcept + : ref(r), number(), object(std::move(v)) {} bool operator<(const Override &rhs) const { return (ref.executor < rhs.ref.executor); @@ -34,29 +45,50 @@ struct Override struct OverrideVisitor : public IPropertiesVisitor { + const BlueprintResolver::ExecutorSpecList &specs; const BlueprintResolver::FeatureMap &feature_map; std::vector<Override> &overrides; - OverrideVisitor(const BlueprintResolver::FeatureMap &feature_map_in, + OverrideVisitor(const BlueprintResolver::ExecutorSpecList &specs_in, + const BlueprintResolver::FeatureMap &feature_map_in, std::vector<Override> &overrides_out) - : feature_map(feature_map_in), overrides(overrides_out) {} + : specs(specs_in), feature_map(feature_map_in), overrides(overrides_out) {} - void visitProperty(const Property::Value & key, - const Property & values) override - { + void visitProperty(const Property::Value &key, const Property &prop) override { auto pos = feature_map.find(key); if (pos != feature_map.end()) { - overrides.emplace_back(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr)); + const auto &name = pos->first; + const auto &ref = pos->second; + const auto &feature_type = specs[ref.executor].output_types[ref.output]; + if (feature_type.is_object()) { + const auto &value_type = feature_type.type(); + const vespalib::string &encoded_value = prop.get(); + vespalib::nbostream stream(encoded_value.data(), encoded_value.size()); + try { + auto tensor = vespalib::eval::decode_value(stream, vespalib::eval::FastValueBuilderFactory::get()); + if (tensor->type() == value_type) { + overrides.emplace_back(ref, std::move(tensor)); + } else { + Issue::report("override for feature '%s' has invalid type: expected %s, got %s", + name.c_str(), value_type.to_spec().c_str(), tensor->type().to_spec().c_str()); + } + } catch (const vespalib::eval::DecodeValueException &e) { + Issue::report("override for feature '%s' has invalid format: %s", name.c_str(), e.what()); + } + } else { + overrides.emplace_back(ref, vespalib::locale::c::strtod(prop.get().c_str(), nullptr)); + } } } }; -std::vector<Override> prepare_overrides(const BlueprintResolver::FeatureMap &feature_map, +std::vector<Override> prepare_overrides(const BlueprintResolver::ExecutorSpecList &specs, + const BlueprintResolver::FeatureMap &feature_map, const Properties &featureOverrides) { std::vector<Override> overrides; overrides.reserve(featureOverrides.numValues()); - OverrideVisitor visitor(feature_map, overrides); + OverrideVisitor visitor(specs, feature_map, overrides); featureOverrides.visitProperties(visitor); std::sort(overrides.begin(), overrides.end()); return overrides; @@ -173,12 +205,12 @@ RankProgram::setup(const MatchData &md, const IQueryEnvironment &queryEnv, const Properties &featureOverrides) { + const auto &specs = _resolver->getExecutorSpecs(); assert(_executors.empty()); - std::vector<Override> overrides = prepare_overrides(_resolver->getFeatureMap(), featureOverrides); + std::vector<Override> overrides = prepare_overrides(specs, _resolver->getFeatureMap(), featureOverrides); auto override = overrides.begin(); auto override_end = overrides.end(); - const auto &specs = _resolver->getExecutorSpecs(); _executors.reserve(specs.size()); _is_const.resize(specs.size()*2); // Reserve space in hashmap for executors to be const for (uint32_t i = 0; i < specs.size(); ++i) { @@ -205,7 +237,7 @@ RankProgram::setup(const MatchData &md, } for (; (override < override_end) && (override->ref.executor == i); ++override) { FeatureExecutor *tmp = executor; - executor = &(stash.get().create<FeatureOverrider>(*tmp, override->ref.output, override->value)); + executor = &(stash.get().create<FeatureOverrider>(*tmp, override->ref.output, override->number, std::move(override->object))); } executor->bind_inputs(inputs); executor->bind_outputs(outputs); diff --git a/searchlib/src/vespa/searchlib/grouping/groupengine.h b/searchlib/src/vespa/searchlib/grouping/groupengine.h index 3a6a680ea1c..778ed2956a8 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupengine.h +++ b/searchlib/src/vespa/searchlib/grouping/groupengine.h @@ -123,15 +123,15 @@ private: size_t getIdBase(GroupRef g) const { return _idByteSize*g; } using IdList = std::unique_ptr<expression::ResultNodeVector>; - typedef vespalib::Array<Children *> GroupBacking; - typedef std::vector<double> RankV; - typedef vespalib::Array<uint8_t> IdBacking; + using GroupBacking = std::vector<Children *>; + using RankV = std::vector<double>; + using IdBacking = std::vector<uint8_t>; const aggregation::GroupingLevel * _request; GroupEngine * _nextEngine; // This is the engine for the next level. size_t _idByteSize; // Correct fixed size of memory needed for one id. IdBacking _ids; // These are all the group ids at this level. - expression::ResultNode::UP _idScratch; // Used for typing the ids. + expression::ResultNode::UP _idScratch; // Used for typing the ids. RankV _rank; // This is the rank of the group. TODO handle with ordinary aggregator. GroupBacking _groupBacking; // These are all the children at this level. Vector<HashTable<GroupRef()>> size_t _level; // This is my level diff --git a/searchlib/src/vespa/searchlib/grouping/groupingengine.h b/searchlib/src/vespa/searchlib/grouping/groupingengine.h index cf5bcaa63f8..b0dbb3f8528 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupingengine.h +++ b/searchlib/src/vespa/searchlib/grouping/groupingengine.h @@ -4,8 +4,7 @@ #include <vespa/searchlib/aggregation/grouping.h> #include <vespa/searchlib/grouping/groupengine.h> -namespace search { -namespace grouping { +namespace search::grouping { class GroupingEngine { @@ -32,4 +31,3 @@ private: }; } -} diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 5a7c8d0f7bc..6fc1ee9e32a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -98,7 +98,7 @@ private: FieldInverter &operator=(const FieldInverter &) = delete; FieldInverter &operator=(const FieldInverter &&) = delete; - using WordBuffer = vespalib::Array<char>; + using WordBuffer = std::vector<char, vespalib::allocator_large<char>>; class ElemInfo { public: diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h index 6ac41c62305..0bf33f1d0e5 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h @@ -3,7 +3,7 @@ #include <memory> #include <cstdint> -#include <vespa/fastos/dynamiclibrary.h> +#include <vespa/fastos/types.h> /** * Interface for posting lists used by PredicateSearch. diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h b/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h index 9707b02ae58..f809d9028c0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h @@ -4,8 +4,7 @@ #include "andsearch.h" -namespace search { -namespace queryeval { +namespace search::queryeval { /** * A simple implementation of the And search operation. @@ -56,6 +55,4 @@ private: Unpack _unpacker; }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h b/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h index 63cbb77dbc8..96fb706f50c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h @@ -3,10 +3,8 @@ #pragma once #include "andsearchnostrict.h" -#include <vespa/fastos/dynamiclibrary.h> -namespace search { -namespace queryeval { +namespace search::queryeval { /** * A simple strict implementation of the And search operation. @@ -105,6 +103,4 @@ AndSearchStrict<Unpack>::andWith(SearchIterator::UP filter, uint32_t estimate_) return filter; // Should always be empty, returning it incase logic changes. } -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp index f259bb3d242..8061160cb5e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp @@ -5,8 +5,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/objects/visit.hpp> -namespace search { -namespace queryeval { +namespace search::queryeval { void BooleanMatchIteratorWrapper::doSeek(uint32_t docid) @@ -41,5 +40,4 @@ BooleanMatchIteratorWrapper::visitMembers(vespalib::ObjectVisitor &visitor) cons // _match not visited } -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/elementiterator.h b/searchlib/src/vespa/searchlib/queryeval/elementiterator.h index 2f46baf087a..aff0fe29779 100644 --- a/searchlib/src/vespa/searchlib/queryeval/elementiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/elementiterator.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchlib/queryeval/searchiterator.h> +#include "searchiterator.h" namespace search::fef { class TermFieldMatchData; } diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h index dfb294c6e0c..897decb456b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h @@ -5,8 +5,7 @@ #include "searchiterator.h" #include <vespa/searchlib/common/bitvector.h> -namespace search { -namespace queryeval { +namespace search::queryeval { /** Search iterator that never yields any hits. */ class EmptySearch : public SearchIterator @@ -28,5 +27,4 @@ public: ~EmptySearch(); }; -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h index 126d395ddb2..706e05f0156 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h @@ -5,8 +5,7 @@ #include "blueprint.h" #include <vespa/searchlib/fef/matchdatalayout.h> -namespace search { -namespace queryeval { +namespace search::queryeval { class EquivBlueprint : public ComplexLeafBlueprint { @@ -32,5 +31,4 @@ public: bool isEquiv() const override { return true; } }; -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_result.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_result.cpp index 8b02dd1d6c9..47e53253210 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_result.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_result.cpp @@ -3,8 +3,7 @@ #include "fake_result.h" #include <ostream> -namespace search { -namespace queryeval { +namespace search::queryeval { FakeResult::FakeResult() : _documents(), @@ -46,5 +45,4 @@ std::ostream &operator << (std::ostream &out, const FakeResult &result) { return out; } -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index 85bbe5ee950..54a48e9ce49 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -3,13 +3,13 @@ #pragma once #include "scores.h" +#include "sorted_hit_sequence.h" #include <vespa/searchlib/common/hitrank.h> #include <vespa/searchlib/common/resultset.h> +#include <vespa/vespalib/util/sort.h> #include <algorithm> #include <vector> -#include <vespa/vespalib/util/sort.h> -#include <vespa/fastos/dynamiclibrary.h> -#include "sorted_hit_sequence.h" +#include <vespa/fastos/types.h> namespace search::queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/matching_elements_search.cpp b/searchlib/src/vespa/searchlib/queryeval/matching_elements_search.cpp index 4a522a00583..a5c3bc8c247 100644 --- a/searchlib/src/vespa/searchlib/queryeval/matching_elements_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/matching_elements_search.cpp @@ -115,7 +115,6 @@ namespace vespalib { template class hash_set<const char *>; template class hashtable<const char *, const char *, hash<const char *>, search::queryeval::EqualCStringValue, Identity, hashtable_base::and_modulator>; -template class Array<vespalib::hash_node<const char *>>; } diff --git a/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp index efbc9d96cc6..a4a72807c0c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp @@ -3,8 +3,7 @@ #include <vespa/log/log.h> LOG_SETUP(".queryeval.monitoring_dump_iterator"); -namespace search { -namespace queryeval { +namespace search::queryeval { MonitoringDumpIterator::MonitoringDumpIterator(MonitoringSearchIterator::UP iterator) : _search(std::move(iterator)) @@ -31,6 +30,4 @@ MonitoringDumpIterator::doUnpack(uint32_t docId) _search->unpack(docId); } -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.h b/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.h index 93229d74a39..7dbe04ac860 100644 --- a/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.h @@ -3,8 +3,7 @@ #include "monitoring_search_iterator.h" -namespace search { -namespace queryeval { +namespace search::queryeval { /** * Search iterator that dumps the search stats of the underlying @@ -29,6 +28,4 @@ public: } }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.cpp index d086d7b2ca5..cff62ea7117 100644 --- a/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.cpp @@ -8,8 +8,7 @@ LOG_SETUP(".queryeval.monitoring_search_iterator"); using vespalib::make_string; -namespace search { -namespace queryeval { +namespace search::queryeval { MonitoringSearchIterator::Stats::Stats() : _numSeeks(0), @@ -237,5 +236,4 @@ MonitoringSearchIterator::visitMembers(vespalib::ObjectVisitor &visitor) const _search->visitMembers(visitor); } -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.h b/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.h index 6345d203f9e..3d20b79a81d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/monitoring_search_iterator.h @@ -5,8 +5,7 @@ #include <vespa/vespalib/objects/objectvisitor.h> #include <stack> -namespace search { -namespace queryeval { +namespace search::queryeval { /** * Search iterator that monitors an underlying search iterator @@ -122,6 +121,4 @@ public: const Stats &getStats() const { return _stats; } }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp index 2966ac06dcc..5297646d7f8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp @@ -8,8 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".nearsearch"); -namespace search { -namespace queryeval { +namespace search::queryeval { namespace { @@ -307,5 +306,4 @@ ONearSearch::match(uint32_t docId) return false; } -} // queryeval -} // search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.h b/searchlib/src/vespa/searchlib/queryeval/nearsearch.h index e2a93171001..bf3fded2717 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.h @@ -5,8 +5,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include "andsearch.h" -namespace search { -namespace queryeval { +namespace search::queryeval { /** * The near search base implements the common logic of the near and o-near search. @@ -145,6 +144,4 @@ public: }; -} // queryeval -} // search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp index f6d20d88a6c..8357d403000 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp @@ -2,7 +2,6 @@ #include "sourceblendersearch.h" #include "isourceselector.h" -#include <vespa/fastos/dynamiclibrary.h> #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/util/array.hpp> diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index da31f1c1a79..1a458f86232 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -51,8 +51,8 @@ public: ++_fail; } ++_done; + _cv.notify_all(); } - _cv.notify_all(); } void diff --git a/storage/src/tests/distributor/btree_bucket_database_test.cpp b/storage/src/tests/distributor/btree_bucket_database_test.cpp index cdeacba4b58..36e700ab1ff 100644 --- a/storage/src/tests/distributor/btree_bucket_database_test.cpp +++ b/storage/src/tests/distributor/btree_bucket_database_test.cpp @@ -2,8 +2,11 @@ #include "bucketdatabasetest.h" #include <vespa/storage/bucketdb/btree_bucket_database.h> +#include <vespa/vespalib/util/count_down_latch.h> +#include <vespa/vespalib/util/time.h> #include <gtest/gtest.h> -#include <gmock/gmock.h> +#include <atomic> +#include <thread> using namespace ::testing; @@ -59,5 +62,77 @@ TEST_F(BTreeReadGuardTest, guard_observes_entries_alive_at_acquire_time) { EXPECT_EQ(entries[0].getBucketInfo(), BI(1, 1234)); } +namespace { + +BucketCopy make_bucket_copy(uint16_t node_idx, uint32_t dummy_info) { + return {0, node_idx, api::BucketInfo(dummy_info, dummy_info, dummy_info)}; +} + +BucketInfo make_bucket_info(uint32_t dummy_info) { + BucketInfo bi; + bi.addNode(make_bucket_copy(0, dummy_info), {0, 1, 2}); + bi.addNode(make_bucket_copy(1, dummy_info), {0, 1, 2}); + bi.addNode(make_bucket_copy(2, dummy_info), {0, 1, 2}); + bi.setLastGarbageCollectionTime(dummy_info); + return bi; +} + +} + +// Simple pseudo-stress test with a single writer and a single reader thread. +// The writer thread continuously updates a set of buckets with an array of bucket +// info instances and last GC timestamp that all have the same value, but the value +// itself is incremented for each write. This allows the reader to validate that it +// is observing a stable snapshot across all read values for a given bucket key. +TEST_F(BTreeReadGuardTest, multithreaded_read_guards_observe_stable_snapshots) { + constexpr uint32_t bucket_bits = 20; + constexpr uint32_t n_buckets = 1u << 10u; // Must be less than 2**bucket_bits + constexpr auto duration = 500ms; + vespalib::CountDownLatch latch(2); + std::atomic<bool> run_reader(true); + + std::thread reader_thread([&]{ + latch.countDown(); + uint32_t read_counter = 0; + while (run_reader.load(std::memory_order_relaxed)) { + auto guard = _db.acquire_read_guard(); + const uint32_t superbucket = (read_counter % n_buckets); + BucketId bucket(bucket_bits, superbucket); + const auto entries = guard->find_parents_and_self(bucket); + // Entry might not have been written yet. If so, yield to give some time. + if (entries.empty()) { + std::this_thread::yield(); + continue; + } + ++read_counter; + // Use plain assertions to avoid any implicit thread/lock interactions with gtest + assert(entries.size() == 1); + const auto& entry = entries[0]; + assert(entry.getBucketId() == bucket); + assert(entry->getNodeCount() == 3); + // We reuse the same write counter as GC timestamp and checksum/doc count/size across + // all stored bucket infos in a given bucket. + const auto expected_stable_val = entry->getLastGarbageCollectionTime(); + for (uint16_t i = 0; i < 3; ++i) { + const auto& info = entry->getNodeRef(i); + assert(info.getChecksum() == expected_stable_val); + assert(info.getDocumentCount() == expected_stable_val); + assert(info.getTotalDocumentSize() == expected_stable_val); + } + } + }); + latch.countDown(); + const auto start_time = vespalib::steady_clock::now(); + uint32_t write_counter = 0; + do { + for (uint32_t i = 0; i < n_buckets; ++i, ++write_counter) { + BucketId bucket_id(bucket_bits, i); + _db.update(BucketDatabase::Entry(bucket_id, make_bucket_info(write_counter))); + } + } while ((vespalib::steady_clock::now() - start_time) < duration); + run_reader.store(false, std::memory_order_relaxed); + reader_thread.join(); +} + } diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index aa55d61e628..a02e98a93b8 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <chrono> #include <vespa/vespalib/gtest/gtest.h> namespace storage::distributor { diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp index 3a1f41cb82c..8ce1ce4ea03 100644 --- a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -67,7 +67,7 @@ FileStorModifiedBucketsTest::modifyBuckets(uint32_t first, uint32_t count) spi::BucketInfo::ACTIVE); } - getDummyPersistence().setModifiedBuckets(buckets); + getDummyPersistence().setModifiedBuckets(std::move(buckets)); } TEST_F(FileStorModifiedBucketsTest, modified_buckets_send_notify_bucket_change) { diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index 55fd2680832..ae1ac2d65b4 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -69,7 +69,7 @@ ModifiedBucketCheckerTest::modifyBuckets(uint32_t count, uint32_t firstBucket) for (uint32_t i = firstBucket; i < firstBucket + count; ++i) { buckets.push_back(document::BucketId(16, i)); } - getDummyPersistence().setModifiedBuckets(buckets); + getDummyPersistence().setModifiedBuckets(std::move(buckets)); } void diff --git a/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.h b/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.h index 77e71bb4f0c..256a54b19b3 100644 --- a/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.h +++ b/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/btree/btree.h> #include <vespa/vespalib/btree/minmaxaggregated.h> #include <vespa/vespalib/btree/minmaxaggrcalc.h> +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> namespace storage::bucketdb { @@ -38,10 +39,11 @@ namespace storage::bucketdb { template <typename DataStoreTraitsT> class GenericBTreeBucketDatabase { public: - using DataStoreType = typename DataStoreTraitsT::DataStoreType; - using ValueType = typename DataStoreTraitsT::ValueType; - using ConstValueRef = typename DataStoreTraitsT::ConstValueRef; - using GenerationHandler = vespalib::GenerationHandler; + using DataStoreType = typename DataStoreTraitsT::DataStoreType; + using ValueType = typename DataStoreTraitsT::ValueType; + using ConstValueRef = typename DataStoreTraitsT::ConstValueRef; + using GenerationHandler = vespalib::GenerationHandler; + using AtomicValueWrapper = vespalib::datastore::AtomicValueWrapper<uint64_t>; struct KeyUsedBitsMinMaxAggrCalc : vespalib::btree::MinMaxAggrCalc { constexpr static bool aggregate_over_values() { return false; } @@ -51,7 +53,18 @@ public: } }; - using BTree = vespalib::btree::BTree<uint64_t, uint64_t, + // Rationale for using an atomic u64 value type: + // It is expected that the set of bucket keys is much less frequently updated than their + // corresponding values. Since values must be stable for concurrent readers, all written values + // are _immutable_ once created. Consequently, every single mutation of a bucket will replace its + // value with a new (immutable) entry. For distributors, this replaces an entire array of values. + // Instead of constantly thawing and freezing subtrees for each bucket update, we atomically + // replace the value to point to a new u32 EntryRef mangled together with an u32 timestamp. + // This means updates that don't change the set of buckets leave the B-tree node structure + // itself entirely untouched. + // This requires great care to be taken when writing and reading to ensure memory visibility. + using BTree = vespalib::btree::BTree<uint64_t, + vespalib::datastore::AtomicValueWrapper<uint64_t>, vespalib::btree::MinMaxAggregated, std::less<>, vespalib::btree::BTreeDefaultTraits, diff --git a/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.hpp b/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.hpp index 7b0d147df0d..9db36e96fc0 100644 --- a/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.hpp +++ b/storage/src/vespa/storage/bucketdb/generic_btree_bucket_database.hpp @@ -77,16 +77,14 @@ GenericBTreeBucketDatabase<DataStoreTraitsT>::entry_from_iterator(const BTreeCon if (!iter.valid()) { return DataStoreTraitsT::make_invalid_value(); } - const auto value = iter.getData(); - std::atomic_thread_fence(std::memory_order_acquire); + const auto value = iter.getData().load_acquire(); return DataStoreTraitsT::unwrap_from_key_value(_store, iter.getKey(), value); } template <typename DataStoreTraitsT> typename GenericBTreeBucketDatabase<DataStoreTraitsT>::ConstValueRef GenericBTreeBucketDatabase<DataStoreTraitsT>::const_value_ref_from_valid_iterator(const BTreeConstIterator& iter) const { - const auto value = iter.getData(); - std::atomic_thread_fence(std::memory_order_acquire); + const auto value = iter.getData().load_acquire(); return DataStoreTraitsT::unwrap_const_ref_from_key_value(_store, iter.getKey(), value); } @@ -304,7 +302,7 @@ bool GenericBTreeBucketDatabase<DataStoreTraitsT>::remove_by_raw_key(uint64_t ke if (!iter.valid()) { return false; } - const auto value = iter.getData(); + const auto value = iter.getData().load_relaxed(); // Called from writer only DataStoreTraitsT::remove_by_wrapped_value(_store, value); _tree.remove(iter); commit_tree_changes(); @@ -324,12 +322,11 @@ bool GenericBTreeBucketDatabase<DataStoreTraitsT>::update_by_raw_key(uint64_t bu auto iter = _tree.lowerBound(bucket_key); const bool pre_existed = (iter.valid() && (iter.getKey() == bucket_key)); if (pre_existed) { - DataStoreTraitsT::remove_by_wrapped_value(_store, iter.getData()); + DataStoreTraitsT::remove_by_wrapped_value(_store, iter.getData().load_relaxed()); // In-place update of value; does not require tree structure modification - std::atomic_thread_fence(std::memory_order_release); // Must ensure visibility when new array ref is observed - iter.writeData(new_value); + iter.getWData().store_release(new_value); // Must ensure visibility when new array ref is observed } else { - _tree.insert(iter, bucket_key, new_value); + _tree.insert(iter, bucket_key, AtomicValueWrapper(new_value)); } commit_tree_changes(); // TODO does publishing a new root imply an implicit memory fence? return pre_existed; @@ -359,18 +356,17 @@ GenericBTreeBucketDatabase<DataStoreTraitsT>::process_update(const BucketId& buc ValueType entry(found ? entry_from_iterator(iter) : processor.create_entry(bucket)); bool keep = processor.process_entry(entry); if (found) { - DataStoreTraitsT::remove_by_wrapped_value(_store, iter.getData()); + DataStoreTraitsT::remove_by_wrapped_value(_store, iter.getData().load_relaxed()); // Called from writer only if (keep) { const auto new_value = DataStoreTraitsT::wrap_and_store_value(_store, entry); - std::atomic_thread_fence(std::memory_order_release); - iter.writeData(new_value); + iter.getWData().store_release(new_value); } else { _tree.remove(iter); } } else { if (keep) { const auto new_value = DataStoreTraitsT::wrap_and_store_value(_store, entry); - _tree.insert(iter, bucket_key, new_value); + _tree.insert(iter, bucket_key, AtomicValueWrapper(new_value)); } } commit_tree_changes(); @@ -491,7 +487,7 @@ struct BTreeBuilderMerger final : Merger<typename DataStoreTraitsT::ValueType> { const uint64_t bucket_key = bucket_id.toKey(); assert(bucket_key < _current_key); const auto new_value = DataStoreTraitsT::wrap_and_store_value(_db.store(), e); - _builder.insert(bucket_key, new_value); + _builder.insert(bucket_key, vespalib::datastore::AtomicValueWrapper<uint64_t>(new_value)); } void update_iteration_state(uint64_t key, uint64_t value) { @@ -520,7 +516,7 @@ struct BTreeTrailingInserter final : TrailingInserter<typename DataStoreTraitsT: void insert_at_end(const BucketId& bucket_id, const ValueType& e) override { const uint64_t bucket_key = bucket_id.toKey(); const auto new_value = DataStoreTraitsT::wrap_and_store_value(_db.store(), e); - _builder.insert(bucket_key, new_value); + _builder.insert(bucket_key, vespalib::datastore::AtomicValueWrapper<uint64_t>(new_value)); } }; @@ -533,19 +529,19 @@ void GenericBTreeBucketDatabase<DataStoreTraitsT>::merge(MergingProcessor<ValueT // TODO for_each instead? for (auto iter = _tree.begin(); iter.valid(); ++iter) { const uint64_t key = iter.getKey(); - const uint64_t value = iter.getData(); + const uint64_t value = iter.getData().load_relaxed(); // Only called from writer merger.update_iteration_state(key, value); auto result = proc.merge(merger); if (result == MergingProcessor<ValueType>::Result::KeepUnchanged) { - builder.insert(key, value); // Reuse array store ref with no changes + builder.insert(key, AtomicValueWrapper(value)); // Reuse array store ref with no changes } else if (result == MergingProcessor<ValueType>::Result::Update) { assert(merger._valid_cached_value); // Must actually have been touched assert(merger._cached_value.valid()); DataStoreTraitsT::remove_by_wrapped_value(_store, value); const auto new_value = DataStoreTraitsT::wrap_and_store_value(_store, merger._cached_value); - builder.insert(key, new_value); + builder.insert(key, AtomicValueWrapper(new_value)); } else if (result == MergingProcessor<ValueType>::Result::Skip) { DataStoreTraitsT::remove_by_wrapped_value(_store, value); } else { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAssertion.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAssertion.java index 23d1b4dfc7e..49cc31fe8c2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAssertion.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAssertion.java @@ -36,6 +36,13 @@ public class AthenzAssertion { return new Builder(role, resource, action); } + public boolean satisfies(AthenzAssertion other) { + return role.equals(other.role()) && + action.equals(other.action()) && + effect().equals(other.effect()) && + resource.equals(other.resource()); + } + public static class Builder { private Long id; private Effect effect; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java index 3ee0c717f19..32efb1d1a39 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/RoleEntity.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * @author mortent @@ -21,16 +22,16 @@ public class RoleEntity { private final List<AuditLogEntry> auditLog; @JsonCreator - public RoleEntity(@JsonProperty("roleName") String roleName, + public RoleEntity(@JsonProperty("name") String roleName, @JsonProperty("roleMembers") List<Member> roleMembers, @JsonProperty("selfServe") Boolean selfServe, @JsonProperty("reviewEnabled") Boolean reviewEnabled, @JsonProperty("auditLog") List<AuditLogEntry> auditLog) { this.roleName = roleName; - this.roleMembers = roleMembers; - this.selfServe = selfServe; - this.reviewEnabled = reviewEnabled; - this.auditLog = auditLog == null ? new ArrayList<>() : auditLog; + this.roleMembers = Optional.ofNullable(roleMembers).orElse(new ArrayList<>()); + this.selfServe = Optional.ofNullable(selfServe).orElse(false); + this.reviewEnabled = Optional.ofNullable(reviewEnabled).orElse(false); + this.auditLog = Optional.ofNullable(auditLog).orElse(new ArrayList<>());; } public String roleName() { diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 7e1fbf5d06a..5f2646ff83c 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -117,11 +117,25 @@ add_valgrind_suppressions_file() { fi } +optionally_reduce_base_frequency() { + if [ -z "$VESPA_TIMER_HZ" ]; then + os_release=`uname -r` + if [[ "$os_release" == *linuxkit* ]]; then + export VESPA_TIMER_HZ=100 + : "Running docker on macos. Reducing base frequency from 1000hz to 100hz due to high cost of sampling time. This will reduce timeout accuracy. VESPA_TIMER_HZ=$VESPA_TIMER_HZ" + fi + else + : "VESPA_TIMER_HZ already set to $VESPA_TIMER_HZ. Skipping auto detection." + fi +} + populate_environment export LD_LIBRARY_PATH=$VESPA_HOME/lib64 export MALLOC_ARENA_MAX=1 +optionally_reduce_base_frequency + # Prefer newer gdb and pstack prepend_path /opt/rh/gcc-toolset-11/root/usr/bin diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index 2c93312eb6d..f035e2c6f00 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -8,6 +8,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -99,7 +100,7 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte /** Returns the items grouped by the given classifier. */ public final <OtherType> Map<OtherType, ListType> groupingBy(Function<Type, OtherType> classifier) { return items.stream().collect(Collectors.groupingBy(classifier, - HashMap::new, + LinkedHashMap::new, Collectors.collectingAndThen(toUnmodifiableList(), (list) -> constructor.apply(list, false)))); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java index 3772aafa044..5532ccbda9c 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java @@ -40,6 +40,8 @@ public final class JsonFormat implements SlimeFormat { } @Override + @Deprecated(since = "7", forRemoval = true) + @SuppressWarnings("removal") public void decode(InputStream is, Slime slime) { throw new UnsupportedOperationException("Not implemented"); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java index 3db659196c9..86003f519bc 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java @@ -19,11 +19,13 @@ public interface SlimeFormat { void encode(OutputStream os, Slime slime) throws IOException; /** - * Encode a slime object into the provided output stream + * Decode an input stream into the provided slime object * * @param is The input stream to read from. * @param slime The slime object to decode into. + * @deprecated use e.g. {@link JsonDecoder} instead */ + @Deprecated(since = "7", forRemoval = true) void decode(InputStream is, Slime slime) throws IOException; } diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java index 763f8cb221c..e06b9826385 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java @@ -227,6 +227,7 @@ public class JsonFormatTestCase { } @Test(expected = UnsupportedOperationException.class) + @SuppressWarnings("removal") public void testThatDecodeIsNotImplemented() { new JsonFormat(true).decode(null, null); } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 6fb832b21d3..29a0c1e7bb8 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -147,6 +147,7 @@ vespa_define_module( src/tests/util/file_area_freelist src/tests/util/generationhandler src/tests/util/generationhandler_stress + src/tests/util/generation_holder src/tests/util/hamming src/tests/util/md5 src/tests/util/mmap_file_allocator diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp index 7d11990577d..4716e91c2c4 100644 --- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp +++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp @@ -196,9 +196,11 @@ protected: using MyTree = typename Params::MyTree; using MyTreeIterator = typename MyTree::Iterator; using MyTreeConstIterator = typename MyTree::ConstIterator; + using KeyStore = IntStore; + using ValueStore = IntStore; GenerationHandler _generationHandler; - IntStore _keys; - IntStore _values; + KeyStore _keys; + ValueStore _values; MyTree _tree; MyTreeIterator _writeItr; vespalib::ThreadStackExecutor _writer; // 1 write thread @@ -344,7 +346,7 @@ template <typename Params> void Fixture<Params>::compact_keys() { - if constexpr (_keys.is_indirect) { + if constexpr (KeyStore::is_indirect) { auto to_hold = _keys.start_compact(); EntryRefFilter filter(_keys.get_num_buffers(), _keys.get_offset_bits()); filter.add_buffers(to_hold); @@ -366,7 +368,7 @@ template <typename Params> void Fixture<Params>::compact_values() { - if constexpr (_values.is_indirect) { + if constexpr (ValueStore::is_indirect) { auto to_hold = _values.start_compact(); EntryRefFilter filter(_values.get_num_buffers(), _values.get_offset_bits()); filter.add_buffers(to_hold); @@ -391,12 +393,12 @@ Fixture<Params>::consider_compact(uint32_t idx) if (_compact_tree.consider(idx) && !_tree.getAllocator().getNodeStore().has_held_buffers()) { compact_tree(); } - if constexpr (_keys.is_indirect) { + if constexpr (KeyStore::is_indirect) { if (_compact_keys.consider(idx) && !_keys.has_held_buffers()) { compact_keys(); } } - if constexpr (_values.is_indirect) { + if constexpr (ValueStore::is_indirect) { if (_compact_values.consider(idx) && !_values.has_held_buffers()) { compact_values(); } diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index 2672ed70f6d..9478ef93327 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -28,9 +28,6 @@ namespace { constexpr float ALLOC_GROW_FACTOR = 0.2; -EntryRef as_entry_ref(const EntryRef& ref) noexcept { return ref; } -EntryRef as_entry_ref(const AtomicEntryRef& ref) noexcept { return ref.load_relaxed(); } - } template <typename EntryT, typename RefT = EntryRefT<19> > @@ -130,22 +127,21 @@ struct Fixture store.transferHoldLists(generation++); store.trimHoldLists(generation); } - template <typename TestedRefType> void compactWorst(bool compactMemory, bool compactAddressSpace) { CompactionSpec compaction_spec(compactMemory, compactAddressSpace); CompactionStrategy compaction_strategy; ICompactionContext::UP ctx = store.compactWorst(compaction_spec, compaction_strategy); - std::vector<TestedRefType> refs; + std::vector<AtomicEntryRef> refs; for (auto itr = refStore.begin(); itr != refStore.end(); ++itr) { refs.emplace_back(itr->first); } - std::vector<TestedRefType> compactedRefs = refs; - ctx->compact(ArrayRef<TestedRefType>(compactedRefs)); + std::vector<AtomicEntryRef> compactedRefs = refs; + ctx->compact(ArrayRef<AtomicEntryRef>(compactedRefs)); ReferenceStore compactedRefStore; for (size_t i = 0; i < refs.size(); ++i) { - ASSERT_EQUAL(0u, compactedRefStore.count(as_entry_ref(compactedRefs[i]))); - ASSERT_EQUAL(1u, refStore.count(as_entry_ref(refs[i]))); - compactedRefStore.insert(std::make_pair(as_entry_ref(compactedRefs[i]), refStore[as_entry_ref(refs[i])])); + ASSERT_EQUAL(0u, compactedRefStore.count(compactedRefs[i].load_relaxed())); + ASSERT_EQUAL(1u, refStore.count(refs[i].load_relaxed())); + compactedRefStore.insert(std::make_pair(compactedRefs[i].load_relaxed(), refStore[refs[i].load_relaxed()])); } refStore = compactedRefStore; } @@ -272,7 +268,6 @@ TEST_F("require that new underlying buffer is allocated when current is full", S namespace { -template <typename TestedRefType> void test_compaction(NumberFixture &f) { @@ -289,7 +284,7 @@ test_compaction(NumberFixture &f) uint32_t size3BufferId = f.getBufferId(size3Ref); EXPECT_EQUAL(3u, f.refStore.size()); - f.compactWorst<TestedRefType>(true, false); + f.compactWorst(true, false); EXPECT_EQUAL(3u, f.refStore.size()); f.assertStoreContent(); @@ -305,14 +300,9 @@ test_compaction(NumberFixture &f) } -TEST_F("require that the buffer with most dead space is compacted (EntryRef vector)", NumberFixture(2)) -{ - test_compaction<EntryRef>(f); -} - -TEST_F("require that the buffer with most dead space is compacted (AtomicEntryRef vector)", NumberFixture(2)) +TEST_F("require that the buffer with most dead space is compacted", NumberFixture(2)) { - test_compaction<AtomicEntryRef>(f); + test_compaction(f); } namespace { @@ -334,7 +324,7 @@ void testCompaction(NumberFixture &f, bool compactMemory, bool compactAddressSpa uint32_t size3BufferId = f.getBufferId(size3Ref); EXPECT_EQUAL(3u, f.refStore.size()); - f.compactWorst<EntryRef>(compactMemory, compactAddressSpace); + f.compactWorst(compactMemory, compactAddressSpace); EXPECT_EQUAL(3u, f.refStore.size()); f.assertStoreContent(); diff --git a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp index a15e9327d16..9f6917d5d65 100644 --- a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp +++ b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp @@ -13,6 +13,14 @@ using namespace vespalib; #endif #endif +#ifndef __SANITIZE_THREAD__ +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define __SANITIZE_THREAD__ +#endif +#endif +#endif + TEST("that uncaught exception causes negative exitcode.") { Process proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught"); EXPECT_LESS(proc.join(), 0); @@ -34,6 +42,7 @@ TEST("that caught silenced exception causes exitcode 0") { } #ifndef __SANITIZE_ADDRESS__ +#ifndef __SANITIZE_THREAD__ #ifdef __APPLE__ // setrlimit with RLIMIT_AS is broken on Darwin #else @@ -53,5 +62,6 @@ TEST("that mmap beyond limits with set VESPA_SILENCE_CORE_ON_OOM cause exitcode } #endif #endif +#endif TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/portal/reactor/reactor_test.cpp b/vespalib/src/tests/portal/reactor/reactor_test.cpp index 82d4d8d3794..d7d61f9ee79 100644 --- a/vespalib/src/tests/portal/reactor/reactor_test.cpp +++ b/vespalib/src/tests/portal/reactor/reactor_test.cpp @@ -42,15 +42,13 @@ void wait_tick() { } } -struct SimpleHandler : Reactor::EventHandler { +struct HandlerBase : Reactor::EventHandler { SocketPair sockets; std::atomic<size_t> read_cnt; std::atomic<size_t> write_cnt; - Reactor::Token::UP token; - SimpleHandler(Reactor &reactor, bool read, bool write) - : sockets(), read_cnt(0), write_cnt(0), token() + HandlerBase() + : sockets(), read_cnt(0), write_cnt(0) { - token = reactor.attach(*this, sockets.main.get(), read, write); } void handle_event(bool read, bool write) override { if (read) { @@ -68,16 +66,33 @@ struct SimpleHandler : Reactor::EventHandler { EXPECT_EQUAL((read_sample != read_cnt), read); EXPECT_EQUAL((write_sample != write_cnt), write); } + ~HandlerBase(); +}; +HandlerBase::~HandlerBase() = default; + +struct SimpleHandler : HandlerBase { + Reactor::Token::UP token; + SimpleHandler(Reactor &reactor, bool read, bool write) + : HandlerBase(), token() + { + token = reactor.attach(*this, sockets.main.get(), read, write); + } ~SimpleHandler(); }; SimpleHandler::~SimpleHandler() = default; -struct DeletingHandler : SimpleHandler { +struct DeletingHandler : HandlerBase { + Gate allow_delete; Gate token_deleted; - DeletingHandler(Reactor &reactor) : SimpleHandler(reactor, true, true), - token_deleted() {} + Reactor::Token::UP token; + DeletingHandler(Reactor &reactor) + : HandlerBase(), allow_delete(), token_deleted(), token() + { + token = reactor.attach(*this, sockets.main.get(), true, true); + } void handle_event(bool read, bool write) override { - SimpleHandler::handle_event(read, write); + HandlerBase::handle_event(read, write); + allow_delete.await(); token.reset(); token_deleted.countDown(); } @@ -85,14 +100,18 @@ struct DeletingHandler : SimpleHandler { }; DeletingHandler::~DeletingHandler() = default; -struct WaitingHandler : SimpleHandler { +struct WaitingHandler : HandlerBase { Gate enter_callback; Gate exit_callback; - WaitingHandler(Reactor &reactor) : SimpleHandler(reactor, true, true), - enter_callback(), exit_callback() {} + Reactor::Token::UP token; + WaitingHandler(Reactor &reactor) + : HandlerBase(), enter_callback(), exit_callback(), token() + { + token = reactor.attach(*this, sockets.main.get(), true, true); + } void handle_event(bool read, bool write) override { enter_callback.countDown(); - SimpleHandler::handle_event(read, write); + HandlerBase::handle_event(read, write); exit_callback.await(); } ~WaitingHandler(); @@ -135,6 +154,7 @@ TEST_FF("require that deleting reactor token disables io events", Reactor(tick), TEST_FF("require that reactor token can be destroyed during io event handling", Reactor(tick), TimeBomb(60)) { DeletingHandler handler(f1); + handler.allow_delete.countDown(); handler.token_deleted.await(); TEST_DO(handler.verify(false, false)); EXPECT_EQUAL(handler.read_cnt, 1u); diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index cd80fa9b984..1f822f9dd6f 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -6,6 +6,8 @@ #include <vespa/vespalib/data/slime/array_value.h> #include <vespa/vespalib/data/slime/strfmt.h> #include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/slime/symbol_table.h> +#include <vespa/vespalib/data/slime/basic_value.h> #include <type_traits> #include <vespa/log/log.h> diff --git a/vespalib/src/tests/util/generation_holder/CMakeLists.txt b/vespalib/src/tests/util/generation_holder/CMakeLists.txt new file mode 100644 index 00000000000..8acf9fadaff --- /dev/null +++ b/vespalib/src/tests/util/generation_holder/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_generation_holder_test_app TEST + SOURCES + generation_holder_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_generation_holder_test_app COMMAND vespalib_generation_holder_test_app) diff --git a/vespalib/src/tests/util/generation_holder/generation_holder_test.cpp b/vespalib/src/tests/util/generation_holder/generation_holder_test.cpp new file mode 100644 index 00000000000..97c3330ac9e --- /dev/null +++ b/vespalib/src/tests/util/generation_holder/generation_holder_test.cpp @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/generationholder.h> + +using vespalib::GenerationHolder; +using MyHeld = vespalib::GenerationHeldBase; + +TEST(GenerationHolderTest, basic_tracking) +{ + GenerationHolder gh; + gh.hold(std::make_unique<MyHeld>(sizeof(int32_t))); + gh.transferHoldLists(0); + gh.hold(std::make_unique<MyHeld>(sizeof(int32_t))); + gh.transferHoldLists(1); + gh.hold(std::make_unique<MyHeld>(sizeof(int32_t))); + gh.transferHoldLists(2); + gh.hold(std::make_unique<MyHeld>(sizeof(int32_t))); + gh.transferHoldLists(4); + EXPECT_EQ(4u * sizeof(int32_t), gh.getHeldBytes()); + gh.trimHoldLists(0); + EXPECT_EQ(4u * sizeof(int32_t), gh.getHeldBytes()); + gh.trimHoldLists(1); + EXPECT_EQ(3u * sizeof(int32_t), gh.getHeldBytes()); + gh.trimHoldLists(2); + EXPECT_EQ(2u * sizeof(int32_t), gh.getHeldBytes()); + gh.hold(std::make_unique<MyHeld>(sizeof(int32_t))); + gh.transferHoldLists(6); + EXPECT_EQ(3u * sizeof(int32_t), gh.getHeldBytes()); + gh.trimHoldLists(6); + EXPECT_EQ(1u * sizeof(int32_t), gh.getHeldBytes()); + gh.trimHoldLists(7); + EXPECT_EQ(0u * sizeof(int32_t), gh.getHeldBytes()); + gh.trimHoldLists(7); + EXPECT_EQ(0u * sizeof(int32_t), gh.getHeldBytes()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/rcuvector/CMakeLists.txt b/vespalib/src/tests/util/rcuvector/CMakeLists.txt index 60e8dc60d43..6b29b73305b 100644 --- a/vespalib/src/tests/util/rcuvector/CMakeLists.txt +++ b/vespalib/src/tests/util/rcuvector/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalib_rcuvector_test_app TEST rcuvector_test.cpp DEPENDS vespalib + GTest::GTest ) vespa_add_test(NAME vespalib_rcuvector_test_app COMMAND vespalib_rcuvector_test_app) diff --git a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp index e36d7da7b6a..58256a96dac 100644 --- a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp +++ b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/rcuvector.h> #include <vespa/vespalib/util/round_up_to_page_size.h> @@ -17,100 +17,71 @@ bool assertUsage(const MemoryUsage & exp, const MemoryUsage & act) { bool retval = true; - if (!EXPECT_EQUAL(exp.allocatedBytes(), act.allocatedBytes())) retval = false; - if (!EXPECT_EQUAL(exp.usedBytes(), act.usedBytes())) retval = false; - if (!EXPECT_EQUAL(exp.deadBytes(), act.deadBytes())) retval = false; - if (!EXPECT_EQUAL(exp.allocatedBytesOnHold(), act.allocatedBytesOnHold())) retval = false; + EXPECT_EQ(exp.allocatedBytes(), act.allocatedBytes()) << (retval = false, ""); + EXPECT_EQ(exp.usedBytes(), act.usedBytes()) << (retval = false, ""); + EXPECT_EQ(exp.deadBytes(), act.deadBytes()) << (retval = false, ""); + EXPECT_EQ(exp.allocatedBytesOnHold(), act.allocatedBytesOnHold()) << ( retval = false, ""); return retval; } -TEST("test generation holder") -{ - GenerationHolder gh; - gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 0)); - gh.transferHoldLists(0); - gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 1)); - gh.transferHoldLists(1); - gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 2)); - gh.transferHoldLists(2); - gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 4)); - gh.transferHoldLists(4); - EXPECT_EQUAL(4u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(0); - EXPECT_EQUAL(4u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(1); - EXPECT_EQUAL(3u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(2); - EXPECT_EQUAL(2u * sizeof(int32_t), gh.getHeldBytes()); - gh.hold(std::make_unique<RcuVectorHeld<int32_t>>(sizeof(int32_t), 6)); - gh.transferHoldLists(6); - EXPECT_EQUAL(3u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(6); - EXPECT_EQUAL(1u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(7); - EXPECT_EQUAL(0u * sizeof(int32_t), gh.getHeldBytes()); - gh.trimHoldLists(7); - EXPECT_EQUAL(0u * sizeof(int32_t), gh.getHeldBytes()); -} - -TEST("test basic") +TEST(RcuVectorTest, basic) { { // insert RcuVector<int32_t> v(4, 0, 4); for (int32_t i = 0; i < 100; ++i) { v.push_back(i); - EXPECT_EQUAL(i, v[i]); - EXPECT_EQUAL((size_t)i + 1, v.size()); + EXPECT_EQ(i, v[i]); + EXPECT_EQ((size_t)i + 1, v.size()); } for (int32_t i = 0; i < 100; ++i) { v[i] = i + 1; - EXPECT_EQUAL(i + 1, v[i]); - EXPECT_EQUAL(100u, v.size()); + EXPECT_EQ(i + 1, v[i]); + EXPECT_EQ(100u, v.size()); } } } -TEST("test resize") +TEST(RcuVectorTest, resize) { { // resize percent RcuVector<int32_t> v(2, 50, 0); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); v.push_back(0); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); v.push_back(0); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); EXPECT_TRUE(v.isFull()); v.push_back(0); - EXPECT_EQUAL(3u, v.capacity()); + EXPECT_EQ(3u, v.capacity()); EXPECT_TRUE(v.isFull()); } { // resize delta RcuVector<int32_t> v(1, 0, 3); - EXPECT_EQUAL(1u, v.capacity()); + EXPECT_EQ(1u, v.capacity()); v.push_back(0); - EXPECT_EQUAL(1u, v.capacity()); + EXPECT_EQ(1u, v.capacity()); EXPECT_TRUE(v.isFull()); v.push_back(0); - EXPECT_EQUAL(4u, v.capacity()); + EXPECT_EQ(4u, v.capacity()); EXPECT_TRUE(!v.isFull()); } { // resize both RcuVector<int32_t> v(2, 200, 3); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); v.push_back(0); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); v.push_back(0); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); EXPECT_TRUE(v.isFull()); v.push_back(0); - EXPECT_EQUAL(9u, v.capacity()); + EXPECT_EQ(9u, v.capacity()); EXPECT_TRUE(!v.isFull()); } { // reserve RcuVector<int32_t> v(2, 0, 0); - EXPECT_EQUAL(2u, v.capacity()); + EXPECT_EQ(2u, v.capacity()); v.unsafe_reserve(8); - EXPECT_EQUAL(8u, v.capacity()); + EXPECT_EQ(8u, v.capacity()); } { // explicit resize GenerationHolder g; @@ -120,71 +91,72 @@ TEST("test resize") g.transferHoldLists(0); g.trimHoldLists(1); const int8_t *old = &v[0]; - EXPECT_EQUAL(16u, v.capacity()); - EXPECT_EQUAL(2u, v.size()); + EXPECT_EQ(16u, v.capacity()); + EXPECT_EQ(2u, v.size()); v.ensure_size(32, 3); v[0] = 3; v[1] = 3; g.transferHoldLists(1); - EXPECT_EQUAL(1, old[0]); - EXPECT_EQUAL(2, old[1]); - EXPECT_EQUAL(3, v[0]); - EXPECT_EQUAL(3, v[1]); - EXPECT_EQUAL(3, v[2]); - EXPECT_EQUAL(3, v[31]); - EXPECT_EQUAL(64u, v.capacity()); - EXPECT_EQUAL(32u, v.size()); + EXPECT_EQ(1, old[0]); + EXPECT_EQ(2, old[1]); + EXPECT_EQ(3, v[0]); + EXPECT_EQ(3, v[1]); + EXPECT_EQ(3, v[2]); + EXPECT_EQ(3, v[31]); + EXPECT_EQ(64u, v.capacity()); + EXPECT_EQ(32u, v.size()); g.trimHoldLists(2); } } -TEST("test generation handling") +TEST(RcuVectorTest, generation_handling) { RcuVector<int32_t> v(2, 0, 2); v.push_back(0); v.push_back(10); - EXPECT_EQUAL(0u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(0u, v.getMemoryUsage().allocatedBytesOnHold()); v.push_back(20); // new array - EXPECT_EQUAL(8u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(8u, v.getMemoryUsage().allocatedBytesOnHold()); v.setGeneration(1); v.push_back(30); - EXPECT_EQUAL(8u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(8u, v.getMemoryUsage().allocatedBytesOnHold()); v.push_back(40); // new array - EXPECT_EQUAL(24u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(24u, v.getMemoryUsage().allocatedBytesOnHold()); v.setGeneration(2); v.push_back(50); v.removeOldGenerations(3); - EXPECT_EQUAL(0u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(0u, v.getMemoryUsage().allocatedBytesOnHold()); v.push_back(60); // new array - EXPECT_EQUAL(24u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(24u, v.getMemoryUsage().allocatedBytesOnHold()); } -TEST("test reserve") { +TEST(RcuVectorTest, reserve) +{ RcuVector<int32_t> v(2, 0, 2); - EXPECT_EQUAL(2u, v.capacity()); - EXPECT_EQUAL(0u, v.size()); + EXPECT_EQ(2u, v.capacity()); + EXPECT_EQ(0u, v.size()); v.push_back(0); v.push_back(10); - EXPECT_EQUAL(2u, v.size()); - EXPECT_EQUAL(2u, v.capacity()); - EXPECT_EQUAL(0u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(2u, v.size()); + EXPECT_EQ(2u, v.capacity()); + EXPECT_EQ(0u, v.getMemoryUsage().allocatedBytesOnHold()); v.reserve(30); - EXPECT_EQUAL(2u, v.size()); - EXPECT_EQUAL(32u, v.capacity()); - EXPECT_EQUAL(8u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(2u, v.size()); + EXPECT_EQ(32u, v.capacity()); + EXPECT_EQ(8u, v.getMemoryUsage().allocatedBytesOnHold()); v.reserve(32); - EXPECT_EQUAL(2u, v.size()); - EXPECT_EQUAL(32u, v.capacity()); - EXPECT_EQUAL(8u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(2u, v.size()); + EXPECT_EQ(32u, v.capacity()); + EXPECT_EQ(8u, v.getMemoryUsage().allocatedBytesOnHold()); v.reserve(100); - EXPECT_EQUAL(2u, v.size()); - EXPECT_EQUAL(102u, v.capacity()); - EXPECT_EQUAL(8u + 32u*4u, v.getMemoryUsage().allocatedBytesOnHold()); + EXPECT_EQ(2u, v.size()); + EXPECT_EQ(102u, v.capacity()); + EXPECT_EQ(8u + 32u*4u, v.getMemoryUsage().allocatedBytesOnHold()); } -TEST("test memory usage") +TEST(RcuVectorTest, memory_usage) { RcuVector<int8_t> v(2, 0, 2); EXPECT_TRUE(assertUsage(MemoryUsage(2,0,0,0), v.getMemoryUsage())); @@ -202,7 +174,7 @@ TEST("test memory usage") EXPECT_TRUE(assertUsage(MemoryUsage(6,5,0,0), v.getMemoryUsage())); } -TEST("test shrink() with buffer copying") +TEST(RcuVectorTest, shrink_with_buffer_copying) { GenerationHolder g; RcuVectorBase<int8_t> v(16, 100, 0, g); @@ -216,24 +188,24 @@ TEST("test shrink() with buffer copying") mu = v.getMemoryUsage(); mu.incAllocatedBytesOnHold(g.getHeldBytes()); EXPECT_TRUE(assertUsage(MemoryUsage(16, 4, 0, 0), mu)); - EXPECT_EQUAL(4u, v.size()); - EXPECT_EQUAL(16u, v.capacity()); - EXPECT_EQUAL(1, v[0]); - EXPECT_EQUAL(2, v[1]); - EXPECT_EQUAL(3, v[2]); - EXPECT_EQUAL(4, v[3]); + EXPECT_EQ(4u, v.size()); + EXPECT_EQ(16u, v.capacity()); + EXPECT_EQ(1, v[0]); + EXPECT_EQ(2, v[1]); + EXPECT_EQ(3, v[2]); + EXPECT_EQ(4, v[3]); const int8_t *old = &v[0]; v.shrink(2); g.transferHoldLists(1); - EXPECT_EQUAL(2u, v.size()); - EXPECT_EQUAL(4u, v.capacity()); - EXPECT_EQUAL(1, v[0]); - EXPECT_EQUAL(2, v[1]); - EXPECT_EQUAL(1, old[0]); - EXPECT_EQUAL(2, old[1]); + EXPECT_EQ(2u, v.size()); + EXPECT_EQ(4u, v.capacity()); + EXPECT_EQ(1, v[0]); + EXPECT_EQ(2, v[1]); + EXPECT_EQ(1, old[0]); + EXPECT_EQ(2, old[1]); g.trimHoldLists(2); - EXPECT_EQUAL(1, v[0]); - EXPECT_EQUAL(2, v[1]); + EXPECT_EQ(1, v[0]); + EXPECT_EQ(2, v[1]); mu = v.getMemoryUsage(); mu.incAllocatedBytesOnHold(g.getHeldBytes()); EXPECT_TRUE(assertUsage(MemoryUsage(4, 2, 0, 0), mu)); @@ -253,51 +225,53 @@ struct ShrinkFixture { for (size_t i = 0; i < initial_size; ++i) { vec.push_back(7); } - EXPECT_EQUAL(initial_size, vec.size()); - EXPECT_EQUAL(initial_capacity, vec.capacity()); + EXPECT_EQ(initial_size, vec.size()); + EXPECT_EQ(initial_capacity, vec.capacity()); assertEmptyHoldList(); oldPtr = &vec[0]; } void assertOldEqualNewBuffer() { - EXPECT_EQUAL(oldPtr, &vec[0]); + EXPECT_EQ(oldPtr, &vec[0]); } void assertEmptyHoldList() { - EXPECT_EQUAL(0u, g.getHeldBytes()); + EXPECT_EQ(0u, g.getHeldBytes()); } static size_t page_ints() { return round_up_to_page_size(1) / sizeof(int); } }; -TEST_F("require that shrink() does not increase allocated memory", ShrinkFixture) +TEST(RcuVectorTest, shrink_does_not_increase_allocated_memory) { + ShrinkFixture f; size_t shrink_size = f.initial_capacity * 2 / 3 + 2; f.vec.shrink(shrink_size); - EXPECT_EQUAL(shrink_size, f.vec.size()); - EXPECT_EQUAL(f.initial_capacity, f.vec.capacity()); - TEST_DO(f.assertOldEqualNewBuffer()); - TEST_DO(f.assertEmptyHoldList()); + EXPECT_EQ(shrink_size, f.vec.size()); + EXPECT_EQ(f.initial_capacity, f.vec.capacity()); + f.assertOldEqualNewBuffer(); + f.assertEmptyHoldList(); } -TEST_F("require that shrink() can shrink mmap allocation", ShrinkFixture) +TEST(RcuVectorTest, shrink_can_shrink_mmap_allocation) { + ShrinkFixture f; f.vec.shrink(2 * f.page_ints()); - EXPECT_EQUAL(2 * f.page_ints(), f.vec.size()); - EXPECT_EQUAL(3 * f.page_ints(), f.vec.capacity()); - TEST_DO(f.assertOldEqualNewBuffer()); - TEST_DO(f.assertEmptyHoldList()); + EXPECT_EQ(2 * f.page_ints(), f.vec.size()); + EXPECT_EQ(3 * f.page_ints(), f.vec.capacity()); + f.assertOldEqualNewBuffer(); + f.assertEmptyHoldList(); } -TEST("test small expand") +TEST(RcuVectorTest, small_expand) { GenerationHolder g; RcuVectorBase<int8_t> v(1, 50, 0, g); - EXPECT_EQUAL(1u, v.capacity()); - EXPECT_EQUAL(0u, v.size()); + EXPECT_EQ(1u, v.capacity()); + EXPECT_EQ(0u, v.size()); v.push_back(1); - EXPECT_EQUAL(1u, v.capacity()); - EXPECT_EQUAL(1u, v.size()); + EXPECT_EQ(1u, v.capacity()); + EXPECT_EQ(1u, v.size()); v.push_back(2); - EXPECT_EQUAL(2u, v.capacity()); - EXPECT_EQUAL(2u, v.size()); + EXPECT_EQ(2u, v.capacity()); + EXPECT_EQ(2u, v.size()); g.transferHoldLists(1); g.trimHoldLists(2); } @@ -332,40 +306,44 @@ Fixture::Fixture() Fixture::~Fixture() = default; -TEST_F("require that memory allocator can be set", Fixture) +TEST(RcuVectorTest, memory_allocator_can_be_set) { - EXPECT_EQUAL(AllocStats(2, 0), f.stats); + Fixture f; + EXPECT_EQ(AllocStats(2, 0), f.stats); f.transfer_and_trim(1, 2); - EXPECT_EQUAL(AllocStats(2, 1), f.stats); + EXPECT_EQ(AllocStats(2, 1), f.stats); } -TEST_F("require that memory allocator is preserved across reset", Fixture) +TEST(RcuVectorTest, memory_allocator_is_preserved_across_reset) { + Fixture f; f.arr.reset(); f.arr.reserve(100); - EXPECT_EQUAL(AllocStats(4, 1), f.stats); + EXPECT_EQ(AllocStats(4, 1), f.stats); f.transfer_and_trim(1, 2); - EXPECT_EQUAL(AllocStats(4, 3), f.stats); + EXPECT_EQ(AllocStats(4, 3), f.stats); } -TEST_F("require that created replacement vector uses same memory allocator", Fixture) +TEST(RcuVectorTest, created_replacement_vector_uses_same_memory_allocator) { + Fixture f; auto arr2 = f.arr.create_replacement_vector(); - EXPECT_EQUAL(AllocStats(2, 0), f.stats); + EXPECT_EQ(AllocStats(2, 0), f.stats); arr2.reserve(100); - EXPECT_EQUAL(AllocStats(3, 0), f.stats); + EXPECT_EQ(AllocStats(3, 0), f.stats); f.transfer_and_trim(1, 2); - EXPECT_EQUAL(AllocStats(3, 1), f.stats); + EXPECT_EQ(AllocStats(3, 1), f.stats); } -TEST_F("require that ensure_size and shrink use same memory allocator", Fixture) +TEST(RcuVectorTest, ensure_size_and_shrink_use_same_memory_allocator) { + Fixture f; f.arr.ensure_size(2000); - EXPECT_EQUAL(AllocStats(3, 0), f.stats); + EXPECT_EQ(AllocStats(3, 0), f.stats); f.arr.shrink(1000); - EXPECT_EQUAL(AllocStats(4, 0), f.stats); + EXPECT_EQ(AllocStats(4, 0), f.stats); f.transfer_and_trim(1, 2); - EXPECT_EQUAL(AllocStats(4, 3), f.stats); + EXPECT_EQ(AllocStats(4, 3), f.stats); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index e0df9744265..e44be0fb5e7 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -5,7 +5,6 @@ #include "btreenode.h" #include "btreenodeallocator.h" #include "btreetraits.h" -#include <vespa/fastos/dynamiclibrary.h> namespace vespalib::btree { diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp index 05323d1329a..9f98ba05493 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.cpp @@ -4,6 +4,7 @@ #include "btreerootbase.h" #include "btreeroot.h" #include "btreenodeallocator.h" +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> #include <vespa/vespalib/datastore/datastore.h> #include <vespa/vespalib/datastore/buffer_type.hpp> @@ -43,6 +44,7 @@ VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint32_t, uint32_t, NoAggrega VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint32_t, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint32_t, int32_t , MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS); VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint64_t, uint64_t , MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS); +VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(uint64_t, AtomicValueWrapper<uint64_t>, MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS); VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(AtomicEntryRef, AtomicEntryRef, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(AtomicEntryRef, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); VESPALIB_DATASTORE_INSTANTIATE_BUFFERTYPE_LEAFNODE(EntryRef, BTreeNoLeafData, NoAggregated, BTreeDefaultTraits::LEAF_SLOTS); diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value.h b/vespalib/src/vespa/vespalib/data/slime/basic_value.h index 43adc39be60..4221c0622a5 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value.h @@ -3,7 +3,6 @@ #pragma once #include "value.h" -#include "memory.h" #include <vespa/vespalib/util/traits.h> namespace vespalib { class Stash; } diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.cpp index db9c21ba0a6..4655f1c36c9 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.cpp @@ -1,7 +1,34 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "basic_value_factory.h" +#include "basic_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { -} // namespace vespalib::slime +Value * +BoolValueFactory::create(Stash & stash) const { + return & stash.create<BasicBoolValue>(input); +} + +Value * +LongValueFactory::create(Stash & stash) const { + return & stash.create<BasicLongValue>(input); +} + +Value * +DoubleValueFactory::create(Stash & stash) const { + return & stash.create<BasicDoubleValue>(input); +} + +Value * +StringValueFactory::create(Stash & stash) const { + return & stash.create<BasicStringValue>(input, stash); +} + +Value * +DataValueFactory::create(Stash & stash) const { + return & stash.create<BasicDataValue>(input, stash); +} + +} diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h index 4fa10fa70f1..451e3fa48cb 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value_factory.h @@ -3,39 +3,38 @@ #pragma once #include "value_factory.h" -#include "basic_value.h" -#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/data/memory.h> namespace vespalib::slime { struct BoolValueFactory final : public ValueFactory { bool input; BoolValueFactory(bool in) noexcept : input(in) {} - Value *create(Stash & stash) const override { return & stash.create<BasicBoolValue>(input); } + Value *create(Stash & stash) const override; }; struct LongValueFactory final : public ValueFactory { int64_t input; LongValueFactory(int64_t in) noexcept : input(in) {} - Value *create(Stash & stash) const override { return & stash.create<BasicLongValue>(input); } + Value *create(Stash & stash) const override; }; struct DoubleValueFactory final : public ValueFactory { double input; DoubleValueFactory(double in) noexcept : input(in) {} - Value *create(Stash & stash) const override { return & stash.create<BasicDoubleValue>(input); } + Value *create(Stash & stash) const override; }; struct StringValueFactory final : public ValueFactory { Memory input; StringValueFactory(Memory in) noexcept : input(in) {} - Value *create(Stash & stash) const override { return & stash.create<BasicStringValue>(input, stash); } + Value *create(Stash & stash) const override; }; struct DataValueFactory final : public ValueFactory { Memory input; DataValueFactory(Memory in) noexcept : input(in) {} - Value *create(Stash & stash) const override { return & stash.create<BasicDataValue>(input, stash); } + Value *create(Stash & stash) const override; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp index a96da032b85..1bb1c642438 100644 --- a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp @@ -3,9 +3,12 @@ #include "external_data_value_factory.h" #include "external_data_value.h" #include "basic_value.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib::slime { +ExternalDataValueFactory::~ExternalDataValueFactory() = default; + Value * ExternalDataValueFactory::create(Stash &stash) const { diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h index b8a85f63084..8d68b430684 100644 --- a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h @@ -3,17 +3,19 @@ #pragma once #include "value_factory.h" -#include "external_memory.h" -#include <vespa/vespalib/util/stash.h> +#include <memory> namespace vespalib::slime { +class ExternalMemory; + /** * Value factory for data values using external memory. **/ struct ExternalDataValueFactory : public ValueFactory { - mutable ExternalMemory::UP input; - ExternalDataValueFactory(ExternalMemory::UP in) : input(std::move(in)) {} + mutable std::unique_ptr<ExternalMemory> input; + ExternalDataValueFactory(std::unique_ptr<ExternalMemory> in) : input(std::move(in)) {} + ~ExternalDataValueFactory() override; Value *create(Stash &stash) const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.h b/vespalib/src/vespa/vespalib/data/slime/json_format.h index fcb3b9d448d..d1e52107526 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.h +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/vespalib/data/output.h> #include "type.h" +#include <vespa/vespalib/data/output.h> #include <vespa/vespalib/data/input_reader.h> #include <vespa/vespalib/data/output_writer.h> diff --git a/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.cpp b/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.cpp index 6905b7034c7..584ec86fe2c 100644 --- a/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.cpp @@ -1,7 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "named_symbol_inserter.h" +#include "symbol_table.h" namespace vespalib::slime { +Symbol +NamedSymbolInserter::insert() { + return _table.insert(_name); +} } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h b/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h index e8bc939332a..0910e0d90e1 100644 --- a/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/named_symbol_inserter.h @@ -3,10 +3,13 @@ #pragma once #include "symbol_inserter.h" -#include "symbol_table.h" + +namespace vespalib { class Memory; } namespace vespalib::slime { +class SymbolTable; + /** * Class used to insert the name of a field into a symbol table. **/ @@ -19,9 +22,7 @@ private: public: NamedSymbolInserter(SymbolTable &table, const Memory &name) noexcept : _table(table), _name(name) {} - Symbol insert() override { - return _table.insert(_name); - } + Symbol insert() override; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.cpp b/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.cpp index 4a0a129ebff..05c6fff67c9 100644 --- a/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.cpp @@ -1,7 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "named_symbol_lookup.h" +#include "symbol_table.h" namespace vespalib::slime { +Symbol +NamedSymbolLookup::lookup() const { + return _table.lookup(_name); +} + } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h b/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h index 685592039cf..6c38551ee72 100644 --- a/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h +++ b/vespalib/src/vespa/vespalib/data/slime/named_symbol_lookup.h @@ -3,10 +3,13 @@ #pragma once #include "symbol_lookup.h" -#include "symbol_table.h" + +namespace vespalib { class Memory; } namespace vespalib::slime { +class SymbolTable; + /** * Class used to look up the name of a field in a symbol table. **/ @@ -19,9 +22,7 @@ private: public: NamedSymbolLookup(const SymbolTable &table, const Memory &name) : _table(table), _name(name) {} - Symbol lookup() const override { - return _table.lookup(_name); - } + Symbol lookup() const override; }; } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.cpp b/vespalib/src/vespa/vespalib/data/slime/object_value.cpp index e8434834d9c..b4bffe4bf00 100644 --- a/vespalib/src/vespa/vespalib/data/slime/object_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/object_value.cpp @@ -6,6 +6,7 @@ #include "resolved_symbol.h" #include "named_symbol_lookup.h" #include "named_symbol_inserter.h" +#include "symbol_table.h" namespace vespalib::slime { diff --git a/vespalib/src/vespa/vespalib/data/slime/root_value.h b/vespalib/src/vespa/vespalib/data/slime/root_value.h index b61325e9cd0..1a7f8115015 100644 --- a/vespalib/src/vespa/vespalib/data/slime/root_value.h +++ b/vespalib/src/vespa/vespalib/data/slime/root_value.h @@ -14,21 +14,21 @@ private: Stash *_stash; public: - RootValue(Stash * stash) : _value(NixValue::instance()), _stash(stash) {} - RootValue(RootValue && rhs) : _value(rhs._value), _stash(rhs._stash) { + RootValue(Stash * stash) noexcept : _value(NixValue::instance()), _stash(stash) {} + RootValue(RootValue && rhs) noexcept : _value(rhs._value), _stash(rhs._stash) { rhs._value = NixValue::instance(); rhs._stash = nullptr; } RootValue(const RootValue &) = delete; RootValue &operator=(const RootValue &) = delete; - RootValue &operator=(RootValue && rhs) { + RootValue &operator=(RootValue && rhs) noexcept { _value = rhs._value; _stash = rhs._stash; rhs._value = NixValue::instance(); rhs._stash = nullptr; return *this; } - Cursor &get() const { return *_value; } + Cursor &get() const noexcept { return *_value; } Cursor &set(const ValueFactory &input) { Value *value = input.create(*_stash); _value = value; diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.cpp b/vespalib/src/vespa/vespalib/data/slime/slime.cpp index 3afcb192f9d..d6fac44b360 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/slime.cpp @@ -1,12 +1,59 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slime.h" +#include "symbol_table.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib { -Slime::~Slime() { } +Slime::Params::Params() : Params(std::make_unique<SymbolTable>()) { } +Slime::Params::Params(std::unique_ptr<SymbolTable> symbols) noexcept : _symbols(std::move(symbols)), _chunkSize(4096) { } +Slime::Params::Params(Params &&) noexcept = default; +Slime::Params::~Params() = default; -bool operator == (const Slime & a, const Slime & b) +std::unique_ptr<slime::SymbolTable> +Slime::Params::detachSymbols() { + return std::move(_symbols); +} + +Slime::Slime(Params params) + : _names(params.detachSymbols()), + _stash(std::make_unique<Stash>(params.getChunkSize())), + _root(_stash.get()) +{ } + +Slime::Slime(Slime &&rhs) noexcept = default; +Slime & Slime::operator=(Slime &&rhs) noexcept = default; +Slime::~Slime() = default; + +std::unique_ptr<slime::SymbolTable> +Slime::reclaimSymbols(Slime &&rhs) { + rhs._stash.reset(); + rhs._root = RootValue(nullptr); + return std::move(rhs._names); +} + +size_t +Slime::symbols() const noexcept { + return _names->symbols(); +} + +Memory +Slime::inspect(Symbol symbol) const { + return _names->inspect(symbol); +} + +slime::Symbol +Slime::insert(Memory name) { + return _names->insert(name); +} + +slime::Symbol +Slime::lookup(Memory name) const { + return _names->lookup(name); +} + +bool operator == (const Slime & a, const Slime & b) noexcept { return a.get() == b.get(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index 6a977b009a4..028133d7be1 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -3,7 +3,6 @@ #pragma once #include "array_traverser.h" -#include "basic_value.h" #include "basic_value_factory.h" #include "binary_format.h" #include "convenience.h" @@ -22,7 +21,6 @@ #include "symbol.h" #include "symbol_inserter.h" #include "symbol_lookup.h" -#include "symbol_table.h" #include "type.h" #include "value.h" #include "value_factory.h" @@ -53,79 +51,54 @@ private: typedef slime::Cursor Cursor; typedef slime::Inspector Inspector; - SymbolTable::UP _names; - Stash::UP _stash; - RootValue _root; + std::unique_ptr<SymbolTable> _names; + std::unique_ptr<Stash> _stash; + RootValue _root; public: typedef std::unique_ptr<Slime> UP; class Params { private: - SymbolTable::UP _symbols; - size_t _chunkSize; + std::unique_ptr<SymbolTable> _symbols; + size_t _chunkSize; public: - Params() : Params(std::make_unique<SymbolTable>()) { } - explicit Params(SymbolTable::UP symbols) : _symbols(std::move(symbols)), _chunkSize(4096) { } + Params(); + explicit Params(std::unique_ptr<SymbolTable> symbols) noexcept; + Params(Params &&) noexcept; + ~Params(); Params & setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; return *this; } size_t getChunkSize() const { return _chunkSize; } - SymbolTable::UP detachSymbols() { return std::move(_symbols); } + std::unique_ptr<SymbolTable> detachSymbols(); }; /** * Construct an initially empty Slime object. **/ - explicit Slime(Params params = Params()) : - _names(params.detachSymbols()), - _stash(std::make_unique<Stash>(params.getChunkSize())), - _root(_stash.get()) - { } + explicit Slime(Params params = Params()); ~Slime(); - Slime(Slime &&rhs) noexcept : - _names(std::move(rhs._names)), - _stash(std::move(rhs._stash)), - _root(std::move(rhs._root)) - { - } + Slime(Slime &&rhs) noexcept; + Slime &operator=(Slime &&rhs) noexcept; Slime(const Slime & rhs) = delete; Slime& operator = (const Slime & rhs) = delete; - static SymbolTable::UP reclaimSymbols(Slime &&rhs) { - rhs._stash.reset(); - rhs._root = RootValue(nullptr); - return std::move(rhs._names); - } - - Slime &operator=(Slime &&rhs) { - _names = std::move(rhs._names); - _stash = std::move(rhs._stash); - _root = std::move(rhs._root); - return *this; - } + static std::unique_ptr<SymbolTable> reclaimSymbols(Slime &&rhs); - size_t symbols() const { - return _names->symbols(); - } + size_t symbols() const noexcept; - Memory inspect(Symbol symbol) const { - return _names->inspect(symbol); - } + Memory inspect(Symbol symbol) const; - Symbol insert(Memory name) { - return _names->insert(name); - } + Symbol insert(Memory name); - Symbol lookup(Memory name) const { - return _names->lookup(name); - } + Symbol lookup(Memory name) const; - Cursor &get() { return _root.get(); } + Cursor &get() noexcept { return _root.get(); } - Inspector &get() const { return _root.get(); } + Inspector &get() const noexcept { return _root.get(); } template <typename ID> Inspector &operator[](ID id) const { return get()[id]; } @@ -177,7 +150,7 @@ public: vespalib::string toString() const { return get().toString(); } }; -bool operator == (const Slime & a, const Slime & b); +bool operator == (const Slime & a, const Slime & b) noexcept; std::ostream & operator << (std::ostream & os, const Slime & slime); } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h index 40200faaca7..2ad983b140c 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h @@ -29,7 +29,7 @@ public: typedef std::unique_ptr<SymbolTable> UP; SymbolTable(size_t expectedNumSymbols=16); ~SymbolTable(); - size_t symbols() const { return _names.size(); } + size_t symbols() const noexcept { return _names.size(); } Memory inspect(const Symbol &symbol) const { if (symbol.getValue() > _names.size()) { return Memory(); diff --git a/vespalib/src/vespa/vespalib/data/slime/type.h b/vespalib/src/vespa/vespalib/data/slime/type.h index 915e80ca7f7..50789d3f53b 100644 --- a/vespalib/src/vespa/vespalib/data/slime/type.h +++ b/vespalib/src/vespa/vespalib/data/slime/type.h @@ -15,10 +15,10 @@ private: uint32_t _id; protected: - Type(uint32_t id) : _id(id) {} + Type(uint32_t id) noexcept : _id(id) {} public: - uint32_t getId() const { return _id; } + uint32_t getId() const noexcept { return _id; } }; /** diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 00c1615b173..f452d0f9b57 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -128,21 +128,11 @@ public: ~CompactionContext() override { _dataStore.finishCompact(_bufferIdsToCompact); } - void compact(vespalib::ArrayRef<EntryRef> refs) override { - for (auto &ref : refs) { - if (ref.valid() && _filter.has(ref)) { - EntryRef newRef = _store.add(_store.get(ref)); - std::atomic_thread_fence(std::memory_order_release); - ref = newRef; - } - } - } void compact(vespalib::ArrayRef<AtomicEntryRef> refs) override { for (auto &atomic_entry_ref : refs) { auto ref = atomic_entry_ref.load_relaxed(); if (ref.valid() && _filter.has(ref)) { EntryRef newRef = _store.add(_store.get(ref)); - std::atomic_thread_fence(std::memory_order_release); atomic_entry_ref.store_release(newRef); } } diff --git a/vespalib/src/vespa/vespalib/datastore/atomic_value_wrapper.h b/vespalib/src/vespa/vespalib/datastore/atomic_value_wrapper.h new file mode 100644 index 00000000000..3ee871be9b0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/atomic_value_wrapper.h @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <atomic> + +namespace vespalib::datastore { + +/** + * Copyable atomic wrapper for a primitive value that offers value store and load + * functionality with explicit memory ordering constraints. Intended to be used for + * non-EntryRef values where atomicity and transitive visibility is a requirement. + * + * Copying always happens with relaxed ordering, as it expects that the copier has + * already loaded the source AtomicValueWrapper with an ordering that is appropriate + * for observing any transitive memory dependencies. + * + * This wrapper is intentionally not implicitly convertible to/from values of the + * underlying primitive type. + * + * Note: use AtomicEntryRef instead if you're wrapping an EntryRef directly. + */ +template <typename T> +class AtomicValueWrapper { + static_assert(std::atomic<T>::is_always_lock_free); + + std::atomic<T> _value; +public: + constexpr AtomicValueWrapper() noexcept : _value() {} + constexpr explicit AtomicValueWrapper(T value) noexcept : _value(value) {} + AtomicValueWrapper(const AtomicValueWrapper& rhs) noexcept + : _value(rhs._value.load(std::memory_order_relaxed)) + {} + AtomicValueWrapper(AtomicValueWrapper&& rhs) noexcept + : _value(rhs._value.load(std::memory_order_relaxed)) + {} + AtomicValueWrapper& operator=(const AtomicValueWrapper& rhs) noexcept { + _value.store(rhs._value.load(std::memory_order_relaxed), + std::memory_order_relaxed); + return *this; + } + void store_release(T value) noexcept { + _value.store(value, std::memory_order_release); + } + void store_relaxed(T value) noexcept { + _value.store(value, std::memory_order_relaxed); + } + [[nodiscard]] T load_acquire() const noexcept { + return _value.load(std::memory_order_acquire); + } + [[nodiscard]] T load_relaxed() const noexcept { + return _value.load(std::memory_order_relaxed); + } + + [[nodiscard]] bool operator==(const AtomicValueWrapper& rhs) const noexcept { + return (load_relaxed() == rhs.load_relaxed()); + } + [[nodiscard]] bool operator!=(const AtomicValueWrapper& rhs) const noexcept { + return !(*this == rhs); + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/i_compaction_context.h b/vespalib/src/vespa/vespalib/datastore/i_compaction_context.h index d4854c4988b..684ad826f20 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_compaction_context.h +++ b/vespalib/src/vespa/vespalib/datastore/i_compaction_context.h @@ -15,7 +15,6 @@ namespace vespalib::datastore { struct ICompactionContext { using UP = std::unique_ptr<ICompactionContext>; virtual ~ICompactionContext() {} - virtual void compact(vespalib::ArrayRef<EntryRef> refs) = 0; virtual void compact(vespalib::ArrayRef<AtomicEntryRef> refs) = 0; }; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index 2e6bccd1857..f5d87e14802 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -3,7 +3,7 @@ #pragma once #include "private_helpers.hpp" -#include <vespa/fastos/dynamiclibrary.h> +#include <vespa/fastos/types.h> namespace vespalib::hwaccelrated::avx { diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp index 3ab300c8f1b..5a843d6774c 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp @@ -81,7 +81,6 @@ hash_map<K, V, H, EQ, M>::getMemoryUsed() const vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&); #define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \ - template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; \ VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::prime_modulator) \ VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::and_modulator) diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp index 2a0617d7d45..332f0c8df69 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp @@ -90,11 +90,9 @@ hash_set<K, H, EQ, M>::insert(K &&value) { #define VESPALIB_HASH_SET_INSTANTIATE(K) \ template class vespalib::hash_set<K>; \ - template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<>, vespalib::Identity, vespalib::hashtable_base::and_modulator>; \ - template class vespalib::Array<vespalib::hash_node<K>>; + template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<>, vespalib::Identity, vespalib::hashtable_base::and_modulator>; #define VESPALIB_HASH_SET_INSTANTIATE_H(K, H) \ template class vespalib::hash_set<K, H>; \ - template class vespalib::hashtable<K, K, H, std::equal_to<>, vespalib::Identity, vespalib::hashtable_base::and_modulator>; \ - template class vespalib::Array<vespalib::hash_node<K>>; + template class vespalib::hashtable<K, K, H, std::equal_to<>, vespalib::Identity, vespalib::hashtable_base::and_modulator>; diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index af98ad0beb4..e8921498a9f 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/array.h> +#include "allocator.h" #include <vespa/vespalib/util/traits.h> +#include <vespa/vespalib/util/alloc.h> #include <algorithm> #include <iterator> @@ -102,15 +103,15 @@ class hash_node { public: using next_t=hashtable_base::next_t; enum {npos=-1u, invalid=-2u}; - hash_node() + hash_node() noexcept : _next(invalid) {} - hash_node(const V & node, next_t next=npos) + hash_node(const V & node, next_t next=npos) noexcept(std::is_nothrow_copy_constructible_v<V>) : _next(next) { new (_node) V(node); } - hash_node(V &&node, next_t next=npos) + hash_node(V &&node, next_t next=npos) noexcept : _next(next) { new (_node) V(std::move(node)); @@ -132,14 +133,14 @@ public: } return *this; } - hash_node(const hash_node & rhs) + hash_node(const hash_node & rhs) noexcept(std::is_nothrow_copy_constructible_v<V>) : _next(rhs._next) { if (rhs.valid()) { new (_node) V(rhs.getValue()); } } - hash_node &operator=(const hash_node & rhs) { + hash_node &operator=(const hash_node & rhs) noexcept (std::is_nothrow_copy_constructible_v<V>) { destruct(); if (rhs.valid()) { new (_node) V(rhs.getValue()); @@ -149,25 +150,25 @@ public: } return *this; } - ~hash_node() { + ~hash_node() noexcept { destruct(); } - bool operator == (const hash_node & rhs) const { + bool operator == (const hash_node & rhs) const noexcept { return (_next == rhs._next) && (!valid() || (getValue() == rhs.getValue())); } - V & getValue() { return *reinterpret_cast<V *>(_node); } - const V & getValue() const { return *reinterpret_cast<const V *>(_node); } - next_t getNext() const { return _next; } - void setNext(next_t next) { _next = next; } - void invalidate() { + V & getValue() noexcept { return *reinterpret_cast<V *>(_node); } + const V & getValue() const noexcept { return *reinterpret_cast<const V *>(_node); } + next_t getNext() const noexcept { return _next; } + void setNext(next_t next) noexcept { _next = next; } + void invalidate() noexcept { destruct(); _next = invalid; } - void terminate() { _next = npos; } - bool valid() const { return _next != invalid; } - bool hasNext() const { return valid() && (_next != npos); } + void terminate() noexcept { _next = npos; } + bool valid() const noexcept { return _next != invalid; } + bool hasNext() const noexcept { return valid() && (_next != npos); } private: - void destruct() { + void destruct() noexcept { if constexpr (!can_skip_destruction<V>::value) { if (valid()) { getValue().~V(); @@ -184,7 +185,7 @@ class hashtable : public hashtable_base private: using Node=hash_node<Value>; protected: - using NodeStore = vespalib::Array<Node>; + using NodeStore = std::vector<Node, allocator_large<Node>>; virtual void move(NodeStore && oldStore); public: class const_iterator; diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp index 52ca9e87247..8289b5a5fd4 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp @@ -2,7 +2,6 @@ #pragma once #include "hashtable.h" -#include <vespa/vespalib/util/array.hpp> #include <algorithm> namespace vespalib { @@ -173,7 +172,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insertInternal(V && n const next_t p(_nodes[h].getNext()); const next_t newIdx(_nodes.size()); _nodes[h].setNext(newIdx); - new (_nodes.push_back_fast()) Node(std::forward<V>(node), p); + _nodes.template emplace_back(std::forward<V>(node), p); _count++; return insert_result(iterator(this, newIdx), true); } else { @@ -197,7 +196,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::force_insert(Value && const next_t p(_nodes[h].getNext()); const next_t newIdx(_nodes.size()); _nodes[h].setNext(newIdx); - new (_nodes.push_back_fast()) Node(std::move(value), p); + _nodes.template emplace_back(std::move(value), p); _count++; } else { resize(_nodes.capacity()*2); diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index 50945bf3cf2..11731e69217 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -48,9 +48,9 @@ struct Chunk { Chunk *next; size_t used; Chunk(const Chunk &) = delete; - explicit Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {} - void clear() { used = sizeof(Chunk); } - char *alloc(size_t size, size_t chunk_size) { + explicit Chunk(Chunk *next_in) noexcept : next(next_in), used(sizeof(Chunk)) {} + void clear() noexcept { used = sizeof(Chunk); } + char *alloc(size_t size, size_t chunk_size) noexcept { size_t aligned_size = ((size + (sizeof(char *) - 1)) & ~(sizeof(char *) - 1)); if (used + aligned_size > chunk_size) { @@ -84,7 +84,7 @@ private: size_t _chunk_size; char *do_alloc(size_t size); - bool is_small(size_t size) const { return (size < (_chunk_size / 4)); } + bool is_small(size_t size) const noexcept { return (size < (_chunk_size / 4)); } template <typename T, typename ... Args> T *init_array(char *mem, size_t size, Args && ... args) { @@ -119,10 +119,10 @@ public: stash::Cleanup *_cleanup; stash::Chunk *_chunk; size_t _used; - Mark(stash::Cleanup *cleanup, stash::Chunk *chunk) + Mark(stash::Cleanup *cleanup, stash::Chunk *chunk) noexcept : _cleanup(cleanup), _chunk(chunk), _used(chunk ? chunk->used : 0u) {} public: - Mark() : Mark(nullptr, nullptr) {} + Mark() noexcept : Mark(nullptr, nullptr) {} }; typedef std::unique_ptr<Stash> UP; @@ -137,7 +137,7 @@ public: void clear(); - Mark mark() const { return Mark(_cleanup, _chunks); } + Mark mark() const noexcept { return Mark(_cleanup, _chunks); } void revert(const Mark &mark); size_t count_used() const; diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index f95db5ecc10..af0fe509080 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -6,6 +6,7 @@ #include <iostream> #include <thread> #include <chrono> +#include <atomic> #include <fcntl.h> #include <unistd.h> #include <sys/stat.h> @@ -19,29 +20,29 @@ LOG_SETUP(".threadtest"); class FileThread : public FastOS_Runnable { - bool _done; + std::atomic<bool> _done; string _file; public: FileThread(string file) : _done(false), _file(file) {} void Run(FastOS_ThreadInterface *thread, void *arg) override; - void stop() {_done = true; } + void stop() { _done.store(true, std::memory_order_relaxed); } }; class LoggerThread : public FastOS_Runnable { - bool _done; + std::atomic<bool> _done; public: - bool _useLogBuffer; + std::atomic<bool> _useLogBuffer; LoggerThread() : _done(false), _useLogBuffer(false) {} void Run(FastOS_ThreadInterface *thread, void *arg) override; - void stop() {_done = true; } + void stop() { _done.store(true, std::memory_order_relaxed); } }; void FileThread::Run(FastOS_ThreadInterface *, void *) { unlink(_file.c_str()); - while (!_done) { + while (!_done.load(std::memory_order_relaxed)) { int fd = open(_file.c_str(), O_RDWR | O_CREAT | O_APPEND, 0644); if (fd == -1) { fprintf(stderr, "open failed: %s\n", strerror(errno)); @@ -66,8 +67,8 @@ void LoggerThread::Run(FastOS_ThreadInterface *, void *) { int counter = 0; - while (!_done) { - if (_useLogBuffer) { + while (!_done.load(std::memory_order_relaxed)) { + if (_useLogBuffer.load(std::memory_order_relaxed)) { LOGBM(info, "bla bla bla %u", ++counter); } else { LOG(info, "bla bla bla"); @@ -114,7 +115,7 @@ ThreadTester::Main() } // Then set to use logbuffer and continue for (int i = 0; i < numLoggers; i++) { - loggers[i]->_useLogBuffer = true; + loggers[i]->_useLogBuffer.store(true, std::memory_order_relaxed); } start = steady_clock::now(); while ((steady_clock::now() - start) < 2.5s) { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index d226cfcd43e..ae10502e569 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.curator; import com.google.inject.Inject; import com.yahoo.cloud.config.CuratorConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.path.Path; import com.yahoo.vespa.curator.api.VespaCurator; import com.yahoo.vespa.curator.recipes.CuratorCounter; @@ -34,9 +36,16 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -49,7 +58,7 @@ import java.util.logging.Logger; * @author vegardh * @author bratseth */ -public class Curator implements VespaCurator, AutoCloseable { +public class Curator extends AbstractComponent implements VespaCurator, AutoCloseable { private static final Logger LOG = Logger.getLogger(Curator.class.getName()); private static final File ZK_CLIENT_CONFIG_FILE = new File(Defaults.getDefaults().underVespaHome("conf/zookeeper/zookeeper-client.cfg")); @@ -82,7 +91,6 @@ public class Curator implements VespaCurator, AutoCloseable { } @Inject - // TODO jonmv: Use a Provider for this, due to required shutdown. public Curator(CuratorConfig curatorConfig, @SuppressWarnings("unused") VespaZooKeeperServer server) { // Depends on ZooKeeperServer to make sure it is started first this(ConnectionSpec.create(curatorConfig.server(), @@ -313,7 +321,21 @@ public class Curator implements VespaCurator, AutoCloseable { @Override public void close() { - curatorFramework.close(); + ExecutorService executor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("curator-shutdown")); + Future<?> shutdown = CompletableFuture.runAsync(curatorFramework::close, executor); + try { + shutdown.get(10, TimeUnit.SECONDS); + } + catch (Exception e) { + LOG.log(Level.WARNING, "Failed shutting down curator framework (within 10 seconds)", e); + if (e instanceof InterruptedException) Thread.currentThread().interrupt(); + } + executor.shutdownNow(); + } + + @Override + public void deconstruct() { + close(); } /** diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java index 604419c063d..389df340ca7 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java @@ -29,8 +29,7 @@ public class Reconfigurer extends AbstractComponent { private static final Logger log = java.util.logging.Logger.getLogger(Reconfigurer.class.getName()); - private static final Duration MIN_TIMEOUT = Duration.ofMinutes(3); - private static final Duration NODE_TIMEOUT = Duration.ofMinutes(1); + private static final Duration TIMEOUT = Duration.ofMinutes(3); private final ExponentialBackoff backoff = new ExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)); private final VespaZooKeeperAdmin vespaZooKeeperAdmin; @@ -95,7 +94,7 @@ public class Reconfigurer extends AbstractComponent { "\nServers in new config:" + servers(newConfig)); String connectionSpec = localConnectionSpec(activeConfig); Instant now = Instant.now(); - Duration reconfigTimeout = reconfigTimeout(newConfig.server().size()); + Duration reconfigTimeout = reconfigTimeout(); Instant end = now.plus(reconfigTimeout); // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed for (int attempt = 1; now.isBefore(end); attempt++) { @@ -129,11 +128,10 @@ public class Reconfigurer extends AbstractComponent { Process.logAndDie("Reconfiguration did not complete within timeout " + reconfigTimeout + ". Forcing container shutdown."); } - /** Returns the timeout to use for the given joining server count */ - private static Duration reconfigTimeout(int joiningServers) { + private static Duration reconfigTimeout() { // For reconfig to succeed, the current and resulting ensembles must have a majority. When an ensemble grows and // the joining servers outnumber the existing ones, we have to wait for enough of them to start to have a majority. - return Duration.ofMillis(Math.max(joiningServers * NODE_TIMEOUT.toMillis(), MIN_TIMEOUT.toMillis())); + return TIMEOUT; } private static String localConnectionSpec(ZookeeperServerConfig config) { |