diff options
| author | Jon Bratseth <bratseth@gmail.com> | 2022-04-07 13:58:39 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@gmail.com> | 2022-04-07 13:58:39 +0200 |
| commit | b68f4ba22e523c39e0ca734d7c8627477518c553 (patch) | |
| tree | 6968efa48259acb6788c2edfab0858de8c8f4765 | |
| parent | a6656689f92f2bcd45c07491aada64740669d5c5 (diff) | |
| parent | 4c3de59b341522a53e3ebbf8ad40bd2b12aff86e (diff) | |
Merge branch 'master' into bratseth/inputs
312 files changed, 5083 insertions, 3554 deletions
diff --git a/application-model/pom.xml b/application-model/pom.xml index 3abf9851d5c..7eac247e249 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -23,6 +23,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java index 7d94a0418e4..20aca379015 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/TenantId.java @@ -31,8 +31,6 @@ public class TenantId { return id; } - public TenantName toName() { return TenantName.from(id); } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/application-preprocessor/src/test/java/com/yahoo/application/preprocessor/ApplicationPreprocessorTest.java b/application-preprocessor/src/test/java/com/yahoo/application/preprocessor/ApplicationPreprocessorTest.java index 1fc4d08b405..5039b05b393 100644 --- a/application-preprocessor/src/test/java/com/yahoo/application/preprocessor/ApplicationPreprocessorTest.java +++ b/application-preprocessor/src/test/java/com/yahoo/application/preprocessor/ApplicationPreprocessorTest.java @@ -2,17 +2,15 @@ package com.yahoo.application.preprocessor; import org.junit.Rule; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.util.Optional; - import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; +import java.io.File; +import java.io.IOException; +import java.util.Optional; public class ApplicationPreprocessorTest { diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index b1b03b60ce6..d53c2c94d5c 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -21,12 +21,12 @@ <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> - <version>3.5.0</version> + <version>3.8.5</version> </dependency> <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-archiver</artifactId> - <version>3.5.0</version> + <version>3.5.2</version> </dependency> <dependency> <groupId>org.apache.maven.plugin-tools</groupId> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java index d922b63c24c..bd6151aea9f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/TestBundleDependencyScopeTranslator.java @@ -35,7 +35,10 @@ public class TestBundleDependencyScopeTranslator implements Artifacts.ScopeTrans this.dependencyScopes = dependencyScopes; } - @Override public String scopeOf(Artifact artifact) { return Objects.requireNonNull(dependencyScopes.get(artifact)); } + @Override + public String scopeOf(Artifact artifact) { + return Objects.requireNonNull(dependencyScopes.get(artifact), () -> "Could not lookup scope for " + artifact); + } public static TestBundleDependencyScopeTranslator from(Map<String, Artifact> dependencies, String rawConfig) { List<DependencyOverride> dependencyOverrides = toDependencyOverrides(rawConfig); diff --git a/client/go/cmd/cert_test.go b/client/go/cmd/cert_test.go index e5837170d15..ee0c21adaf5 100644 --- a/client/go/cmd/cert_test.go +++ b/client/go/cmd/cert_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -25,7 +26,7 @@ func TestCert(t *testing.T) { } func testCert(t *testing.T, subcommand []string) { - pkgDir := mockApplicationPackage(t, false) + appDir, pkgDir := mock.ApplicationPackageDir(t, false, false) cli, stdout, stderr := newTestCLI(t) args := append(subcommand, "-a", "t1.a1.i1", pkgDir) @@ -35,7 +36,6 @@ func testCert(t *testing.T, subcommand []string) { app, err := vespa.ApplicationFromString("t1.a1.i1") assert.Nil(t, err) - appDir := filepath.Join(pkgDir, "src", "main", "application") pkgCertificate := filepath.Join(appDir, "security", "clients.pem") homeDir := cli.config.homeDir certificate := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") @@ -59,7 +59,7 @@ func TestCertCompressedPackage(t *testing.T) { } func testCertCompressedPackage(t *testing.T, subcommand []string) { - pkgDir := mockApplicationPackage(t, true) + _, pkgDir := mock.ApplicationPackageDir(t, true, false) zipFile := filepath.Join(pkgDir, "target", "application.zip") err := os.MkdirAll(filepath.Dir(zipFile), 0755) assert.Nil(t, err) @@ -88,11 +88,10 @@ func TestCertAdd(t *testing.T) { err := cli.Run("auth", "cert", "-N", "-a", "t1.a1.i1") assert.Nil(t, err) - pkgDir := mockApplicationPackage(t, false) + appDir, pkgDir := mock.ApplicationPackageDir(t, false, false) stdout.Reset() err = cli.Run("auth", "cert", "add", "-a", "t1.a1.i1", pkgDir) assert.Nil(t, err) - appDir := filepath.Join(pkgDir, "src", "main", "application") pkgCertificate := filepath.Join(appDir, "security", "clients.pem") assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\n", pkgCertificate), stdout.String()) diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index a287165bb5e..16da69fded6 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -12,12 +12,14 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/version" "github.com/vespa-engine/vespa/client/go/vespa" ) func newDeployCmd(cli *CLI) *cobra.Command { var ( logLevelArg string + versionArg string ) cmd := &cobra.Command{ Use: "deploy [application-directory]", @@ -32,7 +34,12 @@ If application directory is not specified, it defaults to working directory. When deploying to Vespa Cloud the system can be overridden by setting the environment variable VESPA_CLI_CLOUD_SYSTEM. This is intended for internal use -only.`, +only. + +In Vespa Cloud you may override the Vespa runtime version for your deployment. +This option should only be used if you have a reason for using a specific +version. By default Vespa Cloud chooses a suitable version for you. +`, Example: `$ vespa deploy . $ vespa deploy -t cloud $ vespa deploy -t cloud -z dev.aws-us-east-1c # -z can be omitted here as this zone is the default @@ -50,6 +57,13 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return err } opts := cli.createDeploymentOptions(pkg, target) + if versionArg != "" { + version, err := version.Parse(versionArg) + if err != nil { + return err + } + opts.Version = version + } var result vespa.PrepareResult err = cli.spinner(cli.Stderr, "Uploading application package ...", func() error { @@ -75,10 +89,11 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, opts.Target.Deployment().Application.Instance, opts.Target.Deployment().Zone.Environment, opts.Target.Deployment().Zone.Region, result.ID))) } - return waitForQueryService(cli, result.ID) + return waitForQueryService(cli, target, result.ID) }, } cmd.Flags().StringVarP(&logLevelArg, "log-level", "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) + cmd.Flags().StringVarP(&versionArg, "version", "V", "", `Override the Vespa runtime version to use in Vespa Cloud`) return cmd } @@ -142,15 +157,15 @@ func newActivateCmd(cli *CLI) *cobra.Command { return err } cli.printSuccess("Activated ", color.CyanString(pkg.Path), " with session ", sessionID) - return waitForQueryService(cli, sessionID) + return waitForQueryService(cli, target, sessionID) }, } } -func waitForQueryService(cli *CLI, sessionOrRunID int64) error { +func waitForQueryService(cli *CLI, target vespa.Target, sessionOrRunID int64) error { if cli.flags.waitSecs > 0 { log.Println() - _, err := cli.service(vespa.QueryService, sessionOrRunID, "") + _, err := cli.service(target, vespa.QueryService, sessionOrRunID, "") return err } return nil diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index ac135edad4c..32648393492 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -174,7 +174,11 @@ func newDocumentGetCmd(cli *CLI) *cobra.Command { } func documentService(cli *CLI) (*vespa.Service, error) { - return cli.service(vespa.DocumentService, 0, "") + target, err := cli.target(targetOptions{}) + if err != nil { + return nil, err + } + return cli.service(target, vespa.DocumentService, 0, "") } func operationOptions(stderr io.Writer, printCurl bool, timeoutSecs int) vespa.OperationOptions { diff --git a/client/go/cmd/log_test.go b/client/go/cmd/log_test.go index d3e7b630869..d7b5e20fab5 100644 --- a/client/go/cmd/log_test.go +++ b/client/go/cmd/log_test.go @@ -10,7 +10,7 @@ import ( ) func TestLog(t *testing.T) { - pkgDir := mockApplicationPackage(t, false) + _, pkgDir := mock.ApplicationPackageDir(t, false, false) httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) cli, stdout, stderr := newTestCLI(t) @@ -34,7 +34,7 @@ func TestLogOldClient(t *testing.T) { cli, _, stderr := newTestCLI(t) cli.version = version.MustParse("7.0.0") - pkgDir := mockApplicationPackage(t, false) + _, pkgDir := mock.ApplicationPackageDir(t, false, false) httpClient := &mock.HTTPClient{} httpClient.NextResponseString(200, `{"minVersion": "8.0.0"}`) httpClient.NextResponseString(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index c5868e2f71c..7abf17d5358 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -58,7 +58,11 @@ func printCurl(stderr io.Writer, url string, service *vespa.Service) error { } func query(cli *CLI, arguments []string, timeoutSecs int, curl bool) error { - service, err := cli.service(vespa.QueryService, 0, "") + target, err := cli.target(targetOptions{}) + if err != nil { + return err + } + service, err := cli.service(target, vespa.QueryService, 0, "") if err != nil { return err } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 387c72a1f17..452b1f30834 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -406,19 +406,15 @@ func (c *CLI) system(targetType string) (vespa.System, error) { return vespa.System{}, fmt.Errorf("no default system found for %s target", targetType) } -// service returns the service identified by given name and optionally cluster. This function blocks according to the -// wait period configured in this CLI. The parameter sessionOrRunID specifies either the session ID (local target) or -// run ID (cloud target) to wait for. -func (c *CLI) service(name string, sessionOrRunID int64, cluster string) (*vespa.Service, error) { - t, err := c.target(targetOptions{}) - if err != nil { - return nil, err - } +// service returns the service of given name located at target. If non-empty, cluster specifies a cluster to query. This +// function blocks according to the wait period configured in this CLI. The parameter sessionOrRunID specifies either +// the session ID (local target) or run ID (cloud target) to wait for. +func (c *CLI) service(target vespa.Target, name string, sessionOrRunID int64, cluster string) (*vespa.Service, error) { timeout := time.Duration(c.flags.waitSecs) * time.Second if timeout > 0 { log.Printf("Waiting up to %s %s for %s service to become available ...", color.CyanString(strconv.Itoa(c.flags.waitSecs)), color.CyanString("seconds"), color.CyanString(name)) } - s, err := t.Service(name, timeout, sessionOrRunID, cluster) + s, err := target.Service(name, timeout, sessionOrRunID, cluster) if err != nil { return nil, fmt.Errorf("service '%s' is unavailable: %w", name, err) } diff --git a/client/go/cmd/testutil_test.go b/client/go/cmd/testutil_test.go index 68f79187d3a..e5c69e38e93 100644 --- a/client/go/cmd/testutil_test.go +++ b/client/go/cmd/testutil_test.go @@ -3,7 +3,6 @@ package cmd import ( "bytes" - "os" "path/filepath" "testing" @@ -29,21 +28,3 @@ func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Bu cli.exec = &mock.Exec{} return cli, &stdout, &stderr } - -func mockApplicationPackage(t *testing.T, java bool) string { - dir := t.TempDir() - appDir := filepath.Join(dir, "src", "main", "application") - if err := os.MkdirAll(appDir, 0755); err != nil { - t.Fatal(err) - } - servicesXML := filepath.Join(appDir, "services.xml") - if _, err := os.Create(servicesXML); err != nil { - t.Fatal(err) - } - if java { - if _, err := os.Create(filepath.Join(dir, "pom.xml")); err != nil { - t.Fatal(err) - } - } - return dir -} diff --git a/client/go/mock/vespa.go b/client/go/mock/vespa.go new file mode 100644 index 00000000000..ca09a389360 --- /dev/null +++ b/client/go/mock/vespa.go @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package mock + +import ( + "os" + "path/filepath" + "testing" +) + +// ApplicationPackageDir creates a mock application package directory using test helper t, returning the path to the +// "application" directory and the root directory where it was created. If java is true, create a file that indicates +// this package contains Java code. If cert is true, create an empty certificate file. +func ApplicationPackageDir(t *testing.T, java, cert bool) (string, string) { + t.Helper() + rootDir := t.TempDir() + appDir := filepath.Join(rootDir, "src", "main", "application") + if err := os.MkdirAll(appDir, 0755); err != nil { + t.Fatal(err) + } + servicesXML := filepath.Join(appDir, "services.xml") + if _, err := os.Create(servicesXML); err != nil { + t.Fatal(err) + } + if java { + if _, err := os.Create(filepath.Join(rootDir, "pom.xml")); err != nil { + t.Fatal(err) + } + } + if cert { + securityDir := filepath.Join(appDir, "security") + if err := os.MkdirAll(securityDir, 0755); err != nil { + t.Fatal(err) + } + if _, err := os.Create(filepath.Join(securityDir, "clients.pem")); err != nil { + t.Fatal(err) + } + } + return appDir, rootDir +} diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index d479c86a4c7..4fa0bb5b839 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -12,11 +12,13 @@ import ( "mime/multipart" "net/http" "net/url" + "path/filepath" "strconv" "strings" "time" "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/version" ) var DefaultApplication = ApplicationID{Tenant: "default", Application: "application", Instance: "default"} @@ -42,6 +44,7 @@ type DeploymentOptions struct { Target Target ApplicationPackage ApplicationPackage Timeout time.Duration + Version version.Version HTTPClient util.HTTPClient } @@ -259,21 +262,56 @@ func checkDeploymentOpts(opts DeploymentOptions) error { if opts.Target.Type() == TargetCloud && !opts.ApplicationPackage.HasCertificate() { return fmt.Errorf("%s: missing certificate in package", opts) } + if !opts.IsCloud() && !opts.Version.IsZero() { + return fmt.Errorf("%s: custom runtime version is not supported by %s target", opts, opts.Target.Type()) + } return nil } -func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (PrepareResult, error) { +func newDeploymentRequest(url *url.URL, opts DeploymentOptions) (*http.Request, error) { zipReader, err := opts.ApplicationPackage.zipReader(false) if err != nil { - return PrepareResult{}, err + return nil, err } + var body io.Reader header := http.Header{} - header.Add("Content-Type", "application/zip") - request := &http.Request{ + if opts.IsCloud() { + var buf bytes.Buffer + form := multipart.NewWriter(&buf) + formFile, err := form.CreateFormFile("applicationZip", filepath.Base(opts.ApplicationPackage.Path)) + if err != nil { + return nil, err + } + if _, err := io.Copy(formFile, zipReader); err != nil { + return nil, err + } + if !opts.Version.IsZero() { + deployOptions := fmt.Sprintf(`{"vespaVersion":"%s"}`, opts.Version.String()) + if err := form.WriteField("deployOptions", deployOptions); err != nil { + return nil, err + } + } + if err := form.Close(); err != nil { + return nil, err + } + header.Set("Content-Type", form.FormDataContentType()) + body = &buf + } else { + header.Set("Content-Type", "application/zip") + body = zipReader + } + return &http.Request{ URL: url, Method: "POST", Header: header, - Body: io.NopCloser(zipReader), + Body: io.NopCloser(body), + }, nil +} + +func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (PrepareResult, error) { + request, err := newDeploymentRequest(url, opts) + if err != nil { + return PrepareResult{}, err } service, err := opts.Target.Service(DeployService, opts.Timeout, 0, "") if err != nil { diff --git a/client/go/vespa/deploy_test.go b/client/go/vespa/deploy_test.go index f27a2f2927d..297d028ee91 100644 --- a/client/go/vespa/deploy_test.go +++ b/client/go/vespa/deploy_test.go @@ -2,14 +2,75 @@ package vespa import ( + "io" + "mime" + "mime/multipart" + "net/http" "os" "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/mock" + "github.com/vespa-engine/vespa/client/go/version" ) +func TestDeploy(t *testing.T) { + httpClient := mock.HTTPClient{} + target := LocalTarget(&httpClient) + appDir, _ := mock.ApplicationPackageDir(t, false, false) + opts := DeploymentOptions{ + Target: target, + ApplicationPackage: ApplicationPackage{Path: appDir}, + HTTPClient: &httpClient, + } + _, err := Deploy(opts) + assert.Nil(t, err) + assert.Equal(t, 1, len(httpClient.Requests)) + req := httpClient.LastRequest + assert.Equal(t, "http://127.0.0.1:19071/application/v2/tenant/default/prepareandactivate", req.URL.String()) + assert.Equal(t, "application/zip", req.Header.Get("content-type")) + buf := make([]byte, 5) + req.Body.Read(buf) + assert.Equal(t, "PK\x03\x04\x14", string(buf)) +} + +func TestDeployCloud(t *testing.T) { + httpClient := mock.HTTPClient{} + target := createCloudTarget(t, "http://vespacloud", io.Discard) + cloudTarget, ok := target.(*cloudTarget) + require.True(t, ok) + cloudTarget.httpClient = &httpClient + appDir, _ := mock.ApplicationPackageDir(t, false, true) + opts := DeploymentOptions{ + Target: target, + ApplicationPackage: ApplicationPackage{Path: appDir}, + HTTPClient: &httpClient, + } + _, err := Deploy(opts) + require.Nil(t, err) + assert.Equal(t, 1, len(httpClient.Requests)) + req := httpClient.LastRequest + assert.Equal(t, "http://vespacloud/application/v4/tenant/t1/application/a1/instance/i1/deploy/dev-us-north-1", req.URL.String()) + + values := parseMultiPart(t, req) + zipData := values["applicationZip"] + assert.Equal(t, "PK\x03\x04\x14", string(zipData[:5])) + _, hasDeployOptions := values["deployOptions"] + assert.False(t, hasDeployOptions) + + opts.Version = version.MustParse("1.2.3") + _, err = Deploy(opts) + require.Nil(t, err) + req = httpClient.LastRequest + values = parseMultiPart(t, req) + zipData = values["applicationZip"] + assert.Equal(t, "PK\x03\x04\x14", string(zipData[:5])) + assert.Equal(t, string(values["deployOptions"]), `{"vespaVersion":"1.2.3"}`) +} + func TestApplicationFromString(t *testing.T) { app, err := ApplicationFromString("t1.a1.i1") assert.Nil(t, err) @@ -65,6 +126,7 @@ type pkgFixture struct { } func assertFindApplicationPackage(t *testing.T, zipOrDir string, fixture pkgFixture) { + t.Helper() if fixture.existingFile != "" { writeFile(t, fixture.existingFile) } @@ -77,6 +139,7 @@ func assertFindApplicationPackage(t *testing.T, zipOrDir string, fixture pkgFixt } func writeFile(t *testing.T, name string) { + t.Helper() err := os.MkdirAll(filepath.Dir(name), 0755) assert.Nil(t, err) if !strings.HasSuffix(name, string(os.PathSeparator)) { @@ -84,3 +147,29 @@ func writeFile(t *testing.T, name string) { assert.Nil(t, err) } } + +func parseMultiPart(t *testing.T, req *http.Request) map[string][]byte { + t.Helper() + + mediaType, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")) + require.Nil(t, err) + assert.Equal(t, mediaType, "multipart/form-data") + + values := make(map[string][]byte) + mr := multipart.NewReader(req.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(p) + if err != nil { + t.Fatal(err) + } + values[p.FormName()] = data + } + return values +} diff --git a/client/go/vespa/version.go b/client/go/vespa/version.go deleted file mode 100644 index b20c6d360d7..00000000000 --- a/client/go/vespa/version.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package vespa - -// Version is the Vespa CLI version number -var Version string = "0.0.0-devel" // Overriden by linker flag as part of build diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 21e5de31534..b5281050459 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -236,6 +236,7 @@ <include>org.antlr:antlr-runtime:3.5.2:jar:test</include> <include>org.antlr:antlr4-runtime:4.9.3:jar:test</include> <include>org.apache.commons:commons-exec:1.3:jar:test</include> + <include>org.apache.commons:commons-compress:1.21:jar:test</include> <include>org.apache.commons:commons-math3:3.6.1:jar:test</include> <include>org.apache.httpcomponents.client5:httpclient5:${httpclient5.version}:jar:test</include> <include>org.apache.httpcomponents.core5:httpcore5:${httpclient5.version}:jar:test</include> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a410b025e6b..9453f489029 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -120,7 +120,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean avoidRenamingSummaryFeatures() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}, removeAfter = "7.569") default boolean mergeGroupingResultInSearchInvoker() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean experimentalSdParsing() { return false; } - @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.564") default String adminClusterNodeArchitecture() { return adminClusterArchitecture().name(); } + @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.571") default String adminClusterNodeArchitecture() { return adminClusterArchitecture().name(); } @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } } diff --git a/config-model/.gitignore b/config-model/.gitignore index 4cf50da0853..6edd041cbe8 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -5,3 +5,4 @@ /src/test/integration/*/copy/ /src/test/integration/*/models.generated/ *.cfg.actual +/var/ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index cb1e2e047ad..8366dde383b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -81,7 +81,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; - private boolean experimentalSdParsing = false; + private boolean experimentalSdParsing = true; private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 824bf248b5c..1838f1e36b7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -99,7 +99,7 @@ public class MockApplicationPackage implements ApplicationPackage { @SuppressWarnings("deprecation") // not redundant @Override public String getApplicationName() { - return "mock application"; + return "mock-application"; } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index dad8385e0e1..7acf5557236 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -100,17 +100,21 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } + private void applyRanking(ImmutableSDField field, Attribute attribute) { + Ranking ranking = field.getRanking(); + if (ranking != null && ranking.isFilter()) { + attribute.setEnableBitVectors(true); + attribute.setEnableOnlyBitVector(true); + } + } + private void deriveAttribute(ImmutableSDField field, Attribute fieldAttribute) { Attribute attribute = getAttribute(fieldAttribute.getName()); if (attribute == null) { attributes.put(fieldAttribute.getName(), fieldAttribute); attribute = getAttribute(fieldAttribute.getName()); } - Ranking ranking = field.getRanking(); - if (ranking != null && ranking.isFilter()) { - attribute.setEnableBitVectors(true); - attribute.setEnableOnlyBitVector(true); - } + applyRanking(field, attribute); } private void deriveImportedAttributes(ImmutableSDField field) { @@ -134,6 +138,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } Attribute attribute = field.getAttributes().get(field.getName()); if (attribute != null) { + applyRanking(field, attribute); attributes.put(attribute.getName(), attribute.convertToArray()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java index dee9b648228..92e099bc1fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -159,6 +159,15 @@ public class ConvertParsedFields { for (var structField : parsed.getStructFields()) { convertStructField(field, structField); } + if (parsed.hasLiteral()) { + field.getRanking().setLiteral(true); + } + if (parsed.hasFilter()) { + field.getRanking().setFilter(true); + } + if (parsed.hasNormal()) { + field.getRanking().setNormal(true); + } } private void convertStructField(SDField field, ParsedField parsed) { @@ -196,15 +205,6 @@ public class ConvertParsedFields { summaryField.addDestination("default"); summaryField.setTransform(summaryField.getTransform().bold()); } - if (parsed.hasLiteral()) { - field.getRanking().setLiteral(true); - } - if (parsed.hasFilter()) { - field.getRanking().setFilter(true); - } - if (parsed.hasNormal()) { - field.getRanking().setNormal(true); - } } static void convertSummaryFieldSettings(SummaryField summary, ParsedSummaryField parsed) { diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 630e9f0c097..ba41fa0f5b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -27,29 +27,16 @@ public class DocumentTypes { } public DocumenttypesConfig.Builder produce(DocumentModel model, DocumenttypesConfig.Builder builder) { - /* later: - if (some flag) { - return produceDocTypes(model, builder); - } - */ builder.usev8geopositions(this.useV8GeoPositions); Map<NewDocumentType.Name, NewDocumentType> produced = new HashMap<>(); + var indexMap = new IdxMap(); for (NewDocumentType documentType : model.getDocumentManager().getTypes()) { - produceInheritOrder(documentType, builder, produced); + docTypeInheritOrder(documentType, builder, produced, indexMap); } + indexMap.verifyAllDone(); return builder; } - private void produceInheritOrder(NewDocumentType documentType, DocumenttypesConfig.Builder builder, Map<NewDocumentType.Name, NewDocumentType> produced) { - if (!produced.containsKey(documentType.getFullName())) { - for (NewDocumentType inherited : documentType.getInherited()) { - produceInheritOrder(inherited, builder, produced); - } - buildConfig(documentType, builder); - produced.put(documentType.getFullName(), documentType); - } - } - static private <T> List<T> sortedList(Collection<T> unsorted, Comparator<T> cmp) { var list = new ArrayList<T>(); list.addAll(unsorted); @@ -57,203 +44,6 @@ public class DocumentTypes { return list; } - private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) { - if (documentType == VespaDocumentType.INSTANCE) { - return; - } - DocumenttypesConfig.Documenttype.Builder db = new DocumenttypesConfig.Documenttype.Builder(); - db. - id(documentType.getId()). - name(documentType.getName()). - headerstruct(documentType.getContentStruct().getId()); - Set<Integer> built = new HashSet<>(); - for (NewDocumentType inherited : documentType.getInherited()) { - db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); - markAsBuilt(built, inherited.getAllTypes()); - } - for (DataType dt : sortedList(documentType.getTypes(), (a,b) -> a.getName().compareTo(b.getName()))) { - buildConfig(dt, db, built); - } - for (AnnotationType annotation : sortedList(documentType.getAnnotations(), (a,b) -> a.getName().compareTo(b.getName()))) { - DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder(); - db.annotationtype(atb); - buildConfig(annotation, atb); - } - buildConfig(documentType.getFieldSets(), db); - buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); - builder.documenttype(db); - } - - private void buildConfig(Set<FieldSet> fieldSets, DocumenttypesConfig.Documenttype.Builder db) { - for (FieldSet fs : fieldSets) { - buildConfig(fs, db); - } - } - - private void buildConfig(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) { - db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); - } - - private void markAsBuilt(Set<Integer> built, DataTypeCollection typeCollection) { - for (DataType type : typeCollection.getTypes()) { - built.add(type.getId()); - } - } - - private void buildConfig(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) { - builder. - id(annotation.getId()). - name(annotation.getName()); - DataType dt = annotation.getDataType(); - if (dt != null) { - builder.datatype(dt.getId()); - } - for (AnnotationType inherited : annotation.getInheritedTypes()) { - builder.inherits(new DocumenttypesConfig.Documenttype.Annotationtype.Inherits.Builder().id(inherited.getId())); - } - } - - private void buildConfig(DataType type, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) { - if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && !built.contains(type.getId())) { - built.add(type.getId()); - DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder(); - dataTypeBuilder.id(type.getId()); - if (type instanceof TemporaryUnknownType) { - throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); - } - if (type instanceof OwnedTemporaryType) { - throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); - } - if (type instanceof StructDataType) { - buildConfig((StructDataType) type, dataTypeBuilder, documentBuilder, built); - } else if (type instanceof ArrayDataType) { - buildConfig((ArrayDataType) type, dataTypeBuilder, documentBuilder, built); - } else if (type instanceof WeightedSetDataType) { - buildConfig((WeightedSetDataType) type, dataTypeBuilder, documentBuilder, built); - } else if (type instanceof MapDataType) { - buildConfig((MapDataType) type, dataTypeBuilder, documentBuilder, built); - } else if (type instanceof AnnotationReferenceDataType) { - buildConfig((AnnotationReferenceDataType) type, dataTypeBuilder); - } else if (type instanceof TensorDataType) { - // The type of the tensor is not stored here but instead in each field as detailed type information - // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways) - // without changing the field type and thus requiring data refeed - return; - } else if (type instanceof NewDocumentReferenceDataType) { - var refType = (NewDocumentReferenceDataType) type; - if (refType.isTemporary()) { - throw new IllegalArgumentException("Still temporary: " + refType); - } - buildConfig(refType, documentBuilder); - return; - } else { - return; - } - documentBuilder.datatype(dataTypeBuilder); - } - } - - private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { - for (String fieldName : fieldNames) { - var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); - ib.name(fieldName); - builder.importedfield(ib); - } - } - - private void buildConfig(StructDataType type, - DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, - DocumenttypesConfig.Documenttype.Builder documentBuilder, - Set<Integer> built) { - dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.STRUCT); - DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); - dataTypeBuilder.sstruct(structBuilder); - structBuilder.name(type.getName()); - for (com.yahoo.document.Field field : type.getFields()) { - DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder = - new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(); - builder.name(field.getName()). - id(field.getId()). - datatype(field.getDataType().getId()); - if (field.getDataType() instanceof TensorDataType) { - builder.detailedtype(((TensorDataType) field.getDataType()).getTensorType().toString()); - } - structBuilder.field(builder); - buildConfig(field.getDataType(), documentBuilder, built); - } - } - - private void buildConfig(ArrayDataType type, - DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, - DocumenttypesConfig.Documenttype.Builder documentBuilder, - Set<Integer> built) { - dataTypeBuilder. - type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ARRAY). - array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder(). - element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(type.getNestedType().getId()))); - buildConfig(type.getNestedType(), documentBuilder, built); - } - - private void buildConfig(WeightedSetDataType type, - DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, - DocumenttypesConfig.Documenttype.Builder documentBuilder, - Set<Integer> built) { - dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.WSET). - wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder(). - key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder(). - id(type.getNestedType().getId())). - createifnonexistent(type.createIfNonExistent()). - removeifzero(type.removeIfZero())); - buildConfig(type.getNestedType(), documentBuilder, built); - } - - private void buildConfig(MapDataType type, - DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, - DocumenttypesConfig.Documenttype.Builder documentBuilder, - Set<Integer> built) { - dataTypeBuilder. - type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.MAP). - map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder(). - key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder(). - id(type.getKeyType().getId())). - value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder(). - id(type.getValueType().getId()))); - buildConfig(type.getKeyType(), documentBuilder, built); - buildConfig(type.getValueType(), documentBuilder, built); - } - - private void buildConfig(AnnotationReferenceDataType type, - DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder) { - dataTypeBuilder. - type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ANNOTATIONREF). - annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder(). - annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder(). - id(type.getAnnotationType().getId()))); - } - - private void buildConfig(NewDocumentReferenceDataType type, - DocumenttypesConfig.Documenttype.Builder documentBuilder) { - NewDocumentReferenceDataType refType = type; - DocumenttypesConfig.Documenttype.Referencetype.Builder refBuilder = - new DocumenttypesConfig.Documenttype.Referencetype.Builder(); - refBuilder.id(refType.getId()); - refBuilder.target_type_id(type.getTargetTypeId()); - documentBuilder.referencetype(refBuilder); - } - - // Alternate (new) way to build config: - - private DocumenttypesConfig.Builder produceDocTypes(DocumentModel model, DocumenttypesConfig.Builder builder) { - builder.usev8geopositions(this.useV8GeoPositions); - Map<NewDocumentType.Name, NewDocumentType> produced = new HashMap<>(); - var indexMap = new IdxMap(); - for (NewDocumentType documentType : model.getDocumentManager().getTypes()) { - docTypeInheritOrder(documentType, builder, produced, indexMap); - } - indexMap.verifyAllDone(); - return builder; - } - private void docTypeInheritOrder(NewDocumentType documentType, DocumenttypesConfig.Builder builder, Map<NewDocumentType.Name, NewDocumentType> produced, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java index 47bcc64f663..8b8b9e5f40d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java @@ -13,6 +13,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; * * @author Gunnar Gauslaa Bergem */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class Clients extends ConfigModel { private static final long serialVersionUID = 1L; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 88739e92567..b41d805f2a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -212,8 +212,9 @@ public class StorageGroup { if (group.isPresent() && nodes.isPresent()) throw new IllegalArgumentException("Both <group> and <nodes> is specified: Only one of these tags can be used in the same configuration"); - if (group.isPresent() && (group.get().stringAttribute("name") != null || group.get().integerAttribute("distribution-key") != null)) - deployState.getDeployLogger().logApplicationPackage(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); + if (group.isPresent() && (group.get().integerAttribute("distribution-key") != null)) { + deployState.getDeployLogger().logApplicationPackage(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); + } GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null); StorageGroup storageGroup = owner.isHosted() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index ce2bc351d2b..73bd064b967 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -62,7 +62,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.logging.Level; -import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.config.provision.NodeResources.DiskSpeed; import static com.yahoo.config.provision.NodeResources.StorageType; import static java.util.stream.Collectors.toList; @@ -337,8 +336,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce DeployState deployState, String clusterName) { if (admin.getClusterControllers() == null) { - NodeResources nodeResources = clusterControllerResources - .with(Architecture.valueOf(deployState.featureFlags().adminClusterNodeArchitecture())); + NodeResources nodeResources = clusterControllerResources.with(deployState.featureFlags().adminClusterArchitecture()); NodesSpecification spec = NodesSpecification.requiredFromSharedParents(deployState.zone().environment().isProduction() ? 3 : 1, nodeResources, contentElement, diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 68b11178cc2..9c7ee206da1 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -938,6 +938,7 @@ void structFieldBody(ParsedField field) : { } attribute(field) | matchSettings(field.matchSettings()) | queryCommand(field) | + rank(field) | structField(field) | summaryTo(field) ) } diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index 0501aa58784..7ea2fa42a2a 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -1,577 +1,322 @@ enablecompression false usev8geopositions false -documenttype[0].id -853072901 -documenttype[0].name "types" -documenttype[0].version 0 -documenttype[0].headerstruct 1328581348 -documenttype[0].bodystruct 0 -documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id -1486737430 -documenttype[0].datatype[0].type ARRAY -documenttype[0].datatype[0].array.element.id 2 -documenttype[0].datatype[0].map.key.id 0 -documenttype[0].datatype[0].map.value.id 0 -documenttype[0].datatype[0].wset.key.id 0 -documenttype[0].datatype[0].wset.createifnonexistent false -documenttype[0].datatype[0].wset.removeifzero false -documenttype[0].datatype[0].annotationref.annotation.id 0 -documenttype[0].datatype[0].sstruct.name "" -documenttype[0].datatype[0].sstruct.version 0 -documenttype[0].datatype[0].sstruct.compression.type NONE -documenttype[0].datatype[0].sstruct.compression.level 0 -documenttype[0].datatype[0].sstruct.compression.threshold 95 -documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[1].id 1707615575 -documenttype[0].datatype[1].type ARRAY -documenttype[0].datatype[1].array.element.id -1486737430 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 -documenttype[0].datatype[2].id -794985308 -documenttype[0].datatype[2].type ARRAY -documenttype[0].datatype[2].array.element.id 1707615575 -documenttype[0].datatype[2].map.key.id 0 -documenttype[0].datatype[2].map.value.id 0 -documenttype[0].datatype[2].wset.key.id 0 -documenttype[0].datatype[2].wset.createifnonexistent false -documenttype[0].datatype[2].wset.removeifzero false -documenttype[0].datatype[2].annotationref.annotation.id 0 -documenttype[0].datatype[2].sstruct.name "" -documenttype[0].datatype[2].sstruct.version 0 -documenttype[0].datatype[2].sstruct.compression.type NONE -documenttype[0].datatype[2].sstruct.compression.level 0 -documenttype[0].datatype[2].sstruct.compression.threshold 95 -documenttype[0].datatype[2].sstruct.compression.minsize 200 -documenttype[0].datatype[3].id -372512406 -documenttype[0].datatype[3].type MAP -documenttype[0].datatype[3].array.element.id 0 -documenttype[0].datatype[3].map.key.id 0 -documenttype[0].datatype[3].map.value.id 1707615575 -documenttype[0].datatype[3].wset.key.id 0 -documenttype[0].datatype[3].wset.createifnonexistent false -documenttype[0].datatype[3].wset.removeifzero false -documenttype[0].datatype[3].annotationref.annotation.id 0 -documenttype[0].datatype[3].sstruct.name "" -documenttype[0].datatype[3].sstruct.version 0 -documenttype[0].datatype[3].sstruct.compression.type NONE -documenttype[0].datatype[3].sstruct.compression.level 0 -documenttype[0].datatype[3].sstruct.compression.threshold 95 -documenttype[0].datatype[3].sstruct.compression.minsize 200 -documenttype[0].datatype[4].id 1416345047 -documenttype[0].datatype[4].type ARRAY -documenttype[0].datatype[4].array.element.id -372512406 -documenttype[0].datatype[4].map.key.id 0 -documenttype[0].datatype[4].map.value.id 0 -documenttype[0].datatype[4].wset.key.id 0 -documenttype[0].datatype[4].wset.createifnonexistent false -documenttype[0].datatype[4].wset.removeifzero false -documenttype[0].datatype[4].annotationref.annotation.id 0 -documenttype[0].datatype[4].sstruct.name "" -documenttype[0].datatype[4].sstruct.version 0 -documenttype[0].datatype[4].sstruct.compression.type NONE -documenttype[0].datatype[4].sstruct.compression.level 0 -documenttype[0].datatype[4].sstruct.compression.threshold 95 -documenttype[0].datatype[4].sstruct.compression.minsize 200 -documenttype[0].datatype[5].id 339965458 -documenttype[0].datatype[5].type MAP -documenttype[0].datatype[5].array.element.id 0 -documenttype[0].datatype[5].map.key.id 2 -documenttype[0].datatype[5].map.value.id 2 -documenttype[0].datatype[5].wset.key.id 0 -documenttype[0].datatype[5].wset.createifnonexistent false -documenttype[0].datatype[5].wset.removeifzero false -documenttype[0].datatype[5].annotationref.annotation.id 0 -documenttype[0].datatype[5].sstruct.name "" -documenttype[0].datatype[5].sstruct.version 0 -documenttype[0].datatype[5].sstruct.compression.type NONE -documenttype[0].datatype[5].sstruct.compression.level 0 -documenttype[0].datatype[5].sstruct.compression.threshold 95 -documenttype[0].datatype[5].sstruct.compression.minsize 200 -documenttype[0].datatype[6].id 69621385 -documenttype[0].datatype[6].type ARRAY -documenttype[0].datatype[6].array.element.id 339965458 -documenttype[0].datatype[6].map.key.id 0 -documenttype[0].datatype[6].map.value.id 0 -documenttype[0].datatype[6].wset.key.id 0 -documenttype[0].datatype[6].wset.createifnonexistent false -documenttype[0].datatype[6].wset.removeifzero false -documenttype[0].datatype[6].annotationref.annotation.id 0 -documenttype[0].datatype[6].sstruct.name "" -documenttype[0].datatype[6].sstruct.version 0 -documenttype[0].datatype[6].sstruct.compression.type NONE -documenttype[0].datatype[6].sstruct.compression.level 0 -documenttype[0].datatype[6].sstruct.compression.threshold 95 -documenttype[0].datatype[6].sstruct.compression.minsize 200 -documenttype[0].datatype[7].id 49942803 -documenttype[0].datatype[7].type ARRAY -documenttype[0].datatype[7].array.element.id 16 -documenttype[0].datatype[7].map.key.id 0 -documenttype[0].datatype[7].map.value.id 0 -documenttype[0].datatype[7].wset.key.id 0 -documenttype[0].datatype[7].wset.createifnonexistent false -documenttype[0].datatype[7].wset.removeifzero false -documenttype[0].datatype[7].annotationref.annotation.id 0 -documenttype[0].datatype[7].sstruct.name "" -documenttype[0].datatype[7].sstruct.version 0 -documenttype[0].datatype[7].sstruct.compression.type NONE -documenttype[0].datatype[7].sstruct.compression.level 0 -documenttype[0].datatype[7].sstruct.compression.threshold 95 -documenttype[0].datatype[7].sstruct.compression.minsize 200 -documenttype[0].datatype[8].id -1245117006 -documenttype[0].datatype[8].type ARRAY -documenttype[0].datatype[8].array.element.id 0 -documenttype[0].datatype[8].map.key.id 0 -documenttype[0].datatype[8].map.value.id 0 -documenttype[0].datatype[8].wset.key.id 0 -documenttype[0].datatype[8].wset.createifnonexistent false -documenttype[0].datatype[8].wset.removeifzero false -documenttype[0].datatype[8].annotationref.annotation.id 0 -documenttype[0].datatype[8].sstruct.name "" -documenttype[0].datatype[8].sstruct.version 0 -documenttype[0].datatype[8].sstruct.compression.type NONE -documenttype[0].datatype[8].sstruct.compression.level 0 -documenttype[0].datatype[8].sstruct.compression.threshold 95 -documenttype[0].datatype[8].sstruct.compression.minsize 200 -documenttype[0].datatype[9].id -2092985853 -documenttype[0].datatype[9].type STRUCT -documenttype[0].datatype[9].array.element.id 0 -documenttype[0].datatype[9].map.key.id 0 -documenttype[0].datatype[9].map.value.id 0 -documenttype[0].datatype[9].wset.key.id 0 -documenttype[0].datatype[9].wset.createifnonexistent false -documenttype[0].datatype[9].wset.removeifzero false -documenttype[0].datatype[9].annotationref.annotation.id 0 -documenttype[0].datatype[9].sstruct.name "mystruct" -documenttype[0].datatype[9].sstruct.version 0 -documenttype[0].datatype[9].sstruct.compression.type NONE -documenttype[0].datatype[9].sstruct.compression.level 0 -documenttype[0].datatype[9].sstruct.compression.threshold 95 -documenttype[0].datatype[9].sstruct.compression.minsize 200 -documenttype[0].datatype[9].sstruct.field[0].name "bytearr" -documenttype[0].datatype[9].sstruct.field[0].id 1079701754 -documenttype[0].datatype[9].sstruct.field[0].datatype 49942803 -documenttype[0].datatype[9].sstruct.field[0].detailedtype "" -documenttype[0].datatype[9].sstruct.field[1].name "mymap" -documenttype[0].datatype[9].sstruct.field[1].id 1954178122 -documenttype[0].datatype[9].sstruct.field[1].datatype 339965458 -documenttype[0].datatype[9].sstruct.field[1].detailedtype "" -documenttype[0].datatype[9].sstruct.field[2].name "title" -documenttype[0].datatype[9].sstruct.field[2].id 567626448 -documenttype[0].datatype[9].sstruct.field[2].datatype 2 -documenttype[0].datatype[9].sstruct.field[2].detailedtype "" -documenttype[0].datatype[9].sstruct.field[3].name "structfield" -documenttype[0].datatype[9].sstruct.field[3].id 1726890940 -documenttype[0].datatype[9].sstruct.field[3].datatype 2 -documenttype[0].datatype[9].sstruct.field[3].detailedtype "" -documenttype[0].datatype[10].id 759956026 -documenttype[0].datatype[10].type ARRAY -documenttype[0].datatype[10].array.element.id -2092985853 -documenttype[0].datatype[10].map.key.id 0 -documenttype[0].datatype[10].map.value.id 0 -documenttype[0].datatype[10].wset.key.id 0 -documenttype[0].datatype[10].wset.createifnonexistent false -documenttype[0].datatype[10].wset.removeifzero false -documenttype[0].datatype[10].annotationref.annotation.id 0 -documenttype[0].datatype[10].sstruct.name "" -documenttype[0].datatype[10].sstruct.version 0 -documenttype[0].datatype[10].sstruct.compression.type NONE -documenttype[0].datatype[10].sstruct.compression.level 0 -documenttype[0].datatype[10].sstruct.compression.threshold 95 -documenttype[0].datatype[10].sstruct.compression.minsize 200 -documenttype[0].datatype[11].id 109267174 -documenttype[0].datatype[11].type STRUCT -documenttype[0].datatype[11].array.element.id 0 -documenttype[0].datatype[11].map.key.id 0 -documenttype[0].datatype[11].map.value.id 0 -documenttype[0].datatype[11].wset.key.id 0 -documenttype[0].datatype[11].wset.createifnonexistent false -documenttype[0].datatype[11].wset.removeifzero false -documenttype[0].datatype[11].annotationref.annotation.id 0 -documenttype[0].datatype[11].sstruct.name "sct" -documenttype[0].datatype[11].sstruct.version 0 -documenttype[0].datatype[11].sstruct.compression.type NONE -documenttype[0].datatype[11].sstruct.compression.level 0 -documenttype[0].datatype[11].sstruct.compression.threshold 95 -documenttype[0].datatype[11].sstruct.compression.minsize 200 -documenttype[0].datatype[11].sstruct.field[0].name "s1" -documenttype[0].datatype[11].sstruct.field[0].id 2146820765 -documenttype[0].datatype[11].sstruct.field[0].datatype 2 -documenttype[0].datatype[11].sstruct.field[0].detailedtype "" -documenttype[0].datatype[11].sstruct.field[1].name "s2" -documenttype[0].datatype[11].sstruct.field[1].id 45366795 -documenttype[0].datatype[11].sstruct.field[1].datatype 2 -documenttype[0].datatype[11].sstruct.field[1].detailedtype "" -documenttype[0].datatype[12].id -1244829667 -documenttype[0].datatype[12].type ARRAY -documenttype[0].datatype[12].array.element.id 109267174 -documenttype[0].datatype[12].map.key.id 0 -documenttype[0].datatype[12].map.value.id 0 -documenttype[0].datatype[12].wset.key.id 0 -documenttype[0].datatype[12].wset.createifnonexistent false -documenttype[0].datatype[12].wset.removeifzero false -documenttype[0].datatype[12].annotationref.annotation.id 0 -documenttype[0].datatype[12].sstruct.name "" -documenttype[0].datatype[12].sstruct.version 0 -documenttype[0].datatype[12].sstruct.compression.type NONE -documenttype[0].datatype[12].sstruct.compression.level 0 -documenttype[0].datatype[12].sstruct.compression.threshold 95 -documenttype[0].datatype[12].sstruct.compression.minsize 200 -documenttype[0].datatype[13].id 2138385264 -documenttype[0].datatype[13].type MAP -documenttype[0].datatype[13].array.element.id 0 -documenttype[0].datatype[13].map.key.id 0 -documenttype[0].datatype[13].map.value.id 5 -documenttype[0].datatype[13].wset.key.id 0 -documenttype[0].datatype[13].wset.createifnonexistent false -documenttype[0].datatype[13].wset.removeifzero false -documenttype[0].datatype[13].annotationref.annotation.id 0 -documenttype[0].datatype[13].sstruct.name "" -documenttype[0].datatype[13].sstruct.version 0 -documenttype[0].datatype[13].sstruct.compression.type NONE -documenttype[0].datatype[13].sstruct.compression.level 0 -documenttype[0].datatype[13].sstruct.compression.threshold 95 -documenttype[0].datatype[13].sstruct.compression.minsize 200 -documenttype[0].datatype[14].id -1865479609 -documenttype[0].datatype[14].type MAP -documenttype[0].datatype[14].array.element.id 0 -documenttype[0].datatype[14].map.key.id 2 -documenttype[0].datatype[14].map.value.id 4 -documenttype[0].datatype[14].wset.key.id 0 -documenttype[0].datatype[14].wset.createifnonexistent false -documenttype[0].datatype[14].wset.removeifzero false -documenttype[0].datatype[14].annotationref.annotation.id 0 -documenttype[0].datatype[14].sstruct.name "" -documenttype[0].datatype[14].sstruct.version 0 -documenttype[0].datatype[14].sstruct.compression.type NONE -documenttype[0].datatype[14].sstruct.compression.level 0 -documenttype[0].datatype[14].sstruct.compression.threshold 95 -documenttype[0].datatype[14].sstruct.compression.minsize 200 -documenttype[0].datatype[15].id 294108848 -documenttype[0].datatype[15].type STRUCT -documenttype[0].datatype[15].array.element.id 0 -documenttype[0].datatype[15].map.key.id 0 -documenttype[0].datatype[15].map.value.id 0 -documenttype[0].datatype[15].wset.key.id 0 -documenttype[0].datatype[15].wset.createifnonexistent false -documenttype[0].datatype[15].wset.removeifzero false -documenttype[0].datatype[15].annotationref.annotation.id 0 -documenttype[0].datatype[15].sstruct.name "folder" -documenttype[0].datatype[15].sstruct.version 0 -documenttype[0].datatype[15].sstruct.compression.type NONE -documenttype[0].datatype[15].sstruct.compression.level 0 -documenttype[0].datatype[15].sstruct.compression.threshold 95 -documenttype[0].datatype[15].sstruct.compression.minsize 200 -documenttype[0].datatype[15].sstruct.field[0].name "Version" -documenttype[0].datatype[15].sstruct.field[0].id 64430502 -documenttype[0].datatype[15].sstruct.field[0].datatype 0 -documenttype[0].datatype[15].sstruct.field[0].detailedtype "" -documenttype[0].datatype[15].sstruct.field[1].name "Name" -documenttype[0].datatype[15].sstruct.field[1].id 2002760220 -documenttype[0].datatype[15].sstruct.field[1].datatype 2 -documenttype[0].datatype[15].sstruct.field[1].detailedtype "" -documenttype[0].datatype[15].sstruct.field[2].name "FlagsCounter" -documenttype[0].datatype[15].sstruct.field[2].id 1741227606 -documenttype[0].datatype[15].sstruct.field[2].datatype -1865479609 -documenttype[0].datatype[15].sstruct.field[2].detailedtype "" -documenttype[0].datatype[15].sstruct.field[3].name "anotherfolder" -documenttype[0].datatype[15].sstruct.field[3].id 1582421848 -documenttype[0].datatype[15].sstruct.field[3].datatype 294108848 -documenttype[0].datatype[15].sstruct.field[3].detailedtype "" -documenttype[0].datatype[16].id -389833101 -documenttype[0].datatype[16].type MAP -documenttype[0].datatype[16].array.element.id 0 -documenttype[0].datatype[16].map.key.id 0 -documenttype[0].datatype[16].map.value.id 294108848 -documenttype[0].datatype[16].wset.key.id 0 -documenttype[0].datatype[16].wset.createifnonexistent false -documenttype[0].datatype[16].wset.removeifzero false -documenttype[0].datatype[16].annotationref.annotation.id 0 -documenttype[0].datatype[16].sstruct.name "" -documenttype[0].datatype[16].sstruct.version 0 -documenttype[0].datatype[16].sstruct.compression.type NONE -documenttype[0].datatype[16].sstruct.compression.level 0 -documenttype[0].datatype[16].sstruct.compression.threshold 95 -documenttype[0].datatype[16].sstruct.compression.minsize 200 -documenttype[0].datatype[17].id -1715531035 -documenttype[0].datatype[17].type MAP -documenttype[0].datatype[17].array.element.id 0 -documenttype[0].datatype[17].map.key.id 0 -documenttype[0].datatype[17].map.value.id 4 -documenttype[0].datatype[17].wset.key.id 0 -documenttype[0].datatype[17].wset.createifnonexistent false -documenttype[0].datatype[17].wset.removeifzero false -documenttype[0].datatype[17].annotationref.annotation.id 0 -documenttype[0].datatype[17].sstruct.name "" -documenttype[0].datatype[17].sstruct.version 0 -documenttype[0].datatype[17].sstruct.compression.type NONE -documenttype[0].datatype[17].sstruct.compression.level 0 -documenttype[0].datatype[17].sstruct.compression.threshold 95 -documenttype[0].datatype[17].sstruct.compression.minsize 200 -documenttype[0].datatype[18].id 1901258752 -documenttype[0].datatype[18].type MAP -documenttype[0].datatype[18].array.element.id 0 -documenttype[0].datatype[18].map.key.id 0 -documenttype[0].datatype[18].map.value.id -2092985853 -documenttype[0].datatype[18].wset.key.id 0 -documenttype[0].datatype[18].wset.createifnonexistent false -documenttype[0].datatype[18].wset.removeifzero false -documenttype[0].datatype[18].annotationref.annotation.id 0 -documenttype[0].datatype[18].sstruct.name "" -documenttype[0].datatype[18].sstruct.version 0 -documenttype[0].datatype[18].sstruct.compression.type NONE -documenttype[0].datatype[18].sstruct.compression.level 0 -documenttype[0].datatype[18].sstruct.compression.threshold 95 -documenttype[0].datatype[18].sstruct.compression.minsize 200 -documenttype[0].datatype[19].id 435886609 -documenttype[0].datatype[19].type MAP -documenttype[0].datatype[19].array.element.id 0 -documenttype[0].datatype[19].map.key.id 2 -documenttype[0].datatype[19].map.value.id -1245117006 -documenttype[0].datatype[19].wset.key.id 0 -documenttype[0].datatype[19].wset.createifnonexistent false -documenttype[0].datatype[19].wset.removeifzero false -documenttype[0].datatype[19].annotationref.annotation.id 0 -documenttype[0].datatype[19].sstruct.name "" -documenttype[0].datatype[19].sstruct.version 0 -documenttype[0].datatype[19].sstruct.compression.type NONE -documenttype[0].datatype[19].sstruct.compression.level 0 -documenttype[0].datatype[19].sstruct.compression.threshold 95 -documenttype[0].datatype[19].sstruct.compression.minsize 200 -documenttype[0].datatype[20].id 2125154557 -documenttype[0].datatype[20].type MAP -documenttype[0].datatype[20].array.element.id 0 -documenttype[0].datatype[20].map.key.id 2 -documenttype[0].datatype[20].map.value.id 1 -documenttype[0].datatype[20].wset.key.id 0 -documenttype[0].datatype[20].wset.createifnonexistent false -documenttype[0].datatype[20].wset.removeifzero false -documenttype[0].datatype[20].annotationref.annotation.id 0 -documenttype[0].datatype[20].sstruct.name "" -documenttype[0].datatype[20].sstruct.version 0 -documenttype[0].datatype[20].sstruct.compression.type NONE -documenttype[0].datatype[20].sstruct.compression.level 0 -documenttype[0].datatype[20].sstruct.compression.threshold 95 -documenttype[0].datatype[20].sstruct.compression.minsize 200 -documenttype[0].datatype[21].id -1584287606 -documenttype[0].datatype[21].type MAP -documenttype[0].datatype[21].array.element.id 0 -documenttype[0].datatype[21].map.key.id 2 -documenttype[0].datatype[21].map.value.id 0 -documenttype[0].datatype[21].wset.key.id 0 -documenttype[0].datatype[21].wset.createifnonexistent false -documenttype[0].datatype[21].wset.removeifzero false -documenttype[0].datatype[21].annotationref.annotation.id 0 -documenttype[0].datatype[21].sstruct.name "" -documenttype[0].datatype[21].sstruct.version 0 -documenttype[0].datatype[21].sstruct.compression.type NONE -documenttype[0].datatype[21].sstruct.compression.level 0 -documenttype[0].datatype[21].sstruct.compression.threshold 95 -documenttype[0].datatype[21].sstruct.compression.minsize 200 -documenttype[0].datatype[22].id 1328286588 -documenttype[0].datatype[22].type WSET -documenttype[0].datatype[22].array.element.id 0 -documenttype[0].datatype[22].map.key.id 0 -documenttype[0].datatype[22].map.value.id 0 -documenttype[0].datatype[22].wset.key.id 2 -documenttype[0].datatype[22].wset.createifnonexistent false -documenttype[0].datatype[22].wset.removeifzero false -documenttype[0].datatype[22].annotationref.annotation.id 0 -documenttype[0].datatype[22].sstruct.name "" -documenttype[0].datatype[22].sstruct.version 0 -documenttype[0].datatype[22].sstruct.compression.type NONE -documenttype[0].datatype[22].sstruct.compression.level 0 -documenttype[0].datatype[22].sstruct.compression.threshold 95 -documenttype[0].datatype[22].sstruct.compression.minsize 200 -documenttype[0].datatype[23].id 2065577986 -documenttype[0].datatype[23].type WSET -documenttype[0].datatype[23].array.element.id 0 -documenttype[0].datatype[23].map.key.id 0 -documenttype[0].datatype[23].map.value.id 0 -documenttype[0].datatype[23].wset.key.id 2 -documenttype[0].datatype[23].wset.createifnonexistent true -documenttype[0].datatype[23].wset.removeifzero false -documenttype[0].datatype[23].annotationref.annotation.id 0 -documenttype[0].datatype[23].sstruct.name "" -documenttype[0].datatype[23].sstruct.version 0 -documenttype[0].datatype[23].sstruct.compression.type NONE -documenttype[0].datatype[23].sstruct.compression.level 0 -documenttype[0].datatype[23].sstruct.compression.threshold 95 -documenttype[0].datatype[23].sstruct.compression.minsize 200 -documenttype[0].datatype[24].id 2125328771 -documenttype[0].datatype[24].type WSET -documenttype[0].datatype[24].array.element.id 0 -documenttype[0].datatype[24].map.key.id 0 -documenttype[0].datatype[24].map.value.id 0 -documenttype[0].datatype[24].wset.key.id 2 -documenttype[0].datatype[24].wset.createifnonexistent false -documenttype[0].datatype[24].wset.removeifzero true -documenttype[0].datatype[24].annotationref.annotation.id 0 -documenttype[0].datatype[24].sstruct.name "" -documenttype[0].datatype[24].sstruct.version 0 -documenttype[0].datatype[24].sstruct.compression.type NONE -documenttype[0].datatype[24].sstruct.compression.level 0 -documenttype[0].datatype[24].sstruct.compression.threshold 95 -documenttype[0].datatype[24].sstruct.compression.minsize 200 -documenttype[0].datatype[25].id 1328581348 -documenttype[0].datatype[25].type STRUCT -documenttype[0].datatype[25].array.element.id 0 -documenttype[0].datatype[25].map.key.id 0 -documenttype[0].datatype[25].map.value.id 0 -documenttype[0].datatype[25].wset.key.id 0 -documenttype[0].datatype[25].wset.createifnonexistent false -documenttype[0].datatype[25].wset.removeifzero false -documenttype[0].datatype[25].annotationref.annotation.id 0 -documenttype[0].datatype[25].sstruct.name "types.header" -documenttype[0].datatype[25].sstruct.version 0 -documenttype[0].datatype[25].sstruct.compression.type NONE -documenttype[0].datatype[25].sstruct.compression.level 0 -documenttype[0].datatype[25].sstruct.compression.threshold 95 -documenttype[0].datatype[25].sstruct.compression.minsize 200 -documenttype[0].datatype[25].sstruct.field[0].name "abyte" -documenttype[0].datatype[25].sstruct.field[0].id 110138156 -documenttype[0].datatype[25].sstruct.field[0].datatype 16 -documenttype[0].datatype[25].sstruct.field[0].detailedtype "" -documenttype[0].datatype[25].sstruct.field[1].name "along" -documenttype[0].datatype[25].sstruct.field[1].id 1206464520 -documenttype[0].datatype[25].sstruct.field[1].datatype 4 -documenttype[0].datatype[25].sstruct.field[1].detailedtype "" -documenttype[0].datatype[25].sstruct.field[2].name "arrayfield" -documenttype[0].datatype[25].sstruct.field[2].id 965790107 -documenttype[0].datatype[25].sstruct.field[2].datatype -1245117006 -documenttype[0].datatype[25].sstruct.field[2].detailedtype "" -documenttype[0].datatype[25].sstruct.field[3].name "setfield" -documenttype[0].datatype[25].sstruct.field[3].id 761581914 -documenttype[0].datatype[25].sstruct.field[3].datatype 1328286588 -documenttype[0].datatype[25].sstruct.field[3].detailedtype "" -documenttype[0].datatype[25].sstruct.field[4].name "pos" -documenttype[0].datatype[25].sstruct.field[4].id 1041567475 -documenttype[0].datatype[25].sstruct.field[4].datatype 1381038251 -documenttype[0].datatype[25].sstruct.field[4].detailedtype "" -documenttype[0].datatype[25].sstruct.field[5].name "setfield2" -documenttype[0].datatype[25].sstruct.field[5].id 1066659198 -documenttype[0].datatype[25].sstruct.field[5].datatype 18 -documenttype[0].datatype[25].sstruct.field[5].detailedtype "" -documenttype[0].datatype[25].sstruct.field[6].name "setfield3" -documenttype[0].datatype[25].sstruct.field[6].id 1180155772 -documenttype[0].datatype[25].sstruct.field[6].datatype 2125328771 -documenttype[0].datatype[25].sstruct.field[6].detailedtype "" -documenttype[0].datatype[25].sstruct.field[7].name "setfield4" -documenttype[0].datatype[25].sstruct.field[7].id 1254131631 -documenttype[0].datatype[25].sstruct.field[7].datatype 2065577986 -documenttype[0].datatype[25].sstruct.field[7].detailedtype "" -documenttype[0].datatype[25].sstruct.field[8].name "tagfield" -documenttype[0].datatype[25].sstruct.field[8].id 1653562069 -documenttype[0].datatype[25].sstruct.field[8].datatype 18 -documenttype[0].datatype[25].sstruct.field[8].detailedtype "" -documenttype[0].datatype[25].sstruct.field[9].name "structfield" -documenttype[0].datatype[25].sstruct.field[9].id 486207386 -documenttype[0].datatype[25].sstruct.field[9].datatype 109267174 -documenttype[0].datatype[25].sstruct.field[9].detailedtype "" -documenttype[0].datatype[25].sstruct.field[10].name "structarrayfield" -documenttype[0].datatype[25].sstruct.field[10].id 335048518 -documenttype[0].datatype[25].sstruct.field[10].datatype -1244829667 -documenttype[0].datatype[25].sstruct.field[10].detailedtype "" -documenttype[0].datatype[25].sstruct.field[11].name "stringmapfield" -documenttype[0].datatype[25].sstruct.field[11].id 117465687 -documenttype[0].datatype[25].sstruct.field[11].datatype 339965458 -documenttype[0].datatype[25].sstruct.field[11].detailedtype "" -documenttype[0].datatype[25].sstruct.field[12].name "intmapfield" -documenttype[0].datatype[25].sstruct.field[12].id 121004462 -documenttype[0].datatype[25].sstruct.field[12].datatype -1584287606 -documenttype[0].datatype[25].sstruct.field[12].detailedtype "" -documenttype[0].datatype[25].sstruct.field[13].name "floatmapfield" -documenttype[0].datatype[25].sstruct.field[13].id 1239120925 -documenttype[0].datatype[25].sstruct.field[13].datatype 2125154557 -documenttype[0].datatype[25].sstruct.field[13].detailedtype "" -documenttype[0].datatype[25].sstruct.field[14].name "longmapfield" -documenttype[0].datatype[25].sstruct.field[14].id 477718745 -documenttype[0].datatype[25].sstruct.field[14].datatype -1715531035 -documenttype[0].datatype[25].sstruct.field[14].detailedtype "" -documenttype[0].datatype[25].sstruct.field[15].name "doublemapfield" -documenttype[0].datatype[25].sstruct.field[15].id 877047192 -documenttype[0].datatype[25].sstruct.field[15].datatype 2138385264 -documenttype[0].datatype[25].sstruct.field[15].detailedtype "" -documenttype[0].datatype[25].sstruct.field[16].name "arraymapfield" -documenttype[0].datatype[25].sstruct.field[16].id 1670805928 -documenttype[0].datatype[25].sstruct.field[16].datatype 435886609 -documenttype[0].datatype[25].sstruct.field[16].detailedtype "" -documenttype[0].datatype[25].sstruct.field[17].name "arrarr" -documenttype[0].datatype[25].sstruct.field[17].id 1962567166 -documenttype[0].datatype[25].sstruct.field[17].datatype -794985308 -documenttype[0].datatype[25].sstruct.field[17].detailedtype "" -documenttype[0].datatype[25].sstruct.field[18].name "maparr" -documenttype[0].datatype[25].sstruct.field[18].id 904375219 -documenttype[0].datatype[25].sstruct.field[18].datatype 69621385 -documenttype[0].datatype[25].sstruct.field[18].detailedtype "" -documenttype[0].datatype[25].sstruct.field[19].name "complexarray" -documenttype[0].datatype[25].sstruct.field[19].id 795629533 -documenttype[0].datatype[25].sstruct.field[19].datatype 1416345047 -documenttype[0].datatype[25].sstruct.field[19].detailedtype "" -documenttype[0].datatype[25].sstruct.field[20].name "mystructfield" -documenttype[0].datatype[25].sstruct.field[20].id 1348513378 -documenttype[0].datatype[25].sstruct.field[20].datatype -2092985853 -documenttype[0].datatype[25].sstruct.field[20].detailedtype "" -documenttype[0].datatype[25].sstruct.field[21].name "mystructmap" -documenttype[0].datatype[25].sstruct.field[21].id 1511423250 -documenttype[0].datatype[25].sstruct.field[21].datatype 1901258752 -documenttype[0].datatype[25].sstruct.field[21].detailedtype "" -documenttype[0].datatype[25].sstruct.field[22].name "mystructarr" -documenttype[0].datatype[25].sstruct.field[22].id 595856991 -documenttype[0].datatype[25].sstruct.field[22].datatype 759956026 -documenttype[0].datatype[25].sstruct.field[22].detailedtype "" -documenttype[0].datatype[25].sstruct.field[23].name "Folders" -documenttype[0].datatype[25].sstruct.field[23].id 34575524 -documenttype[0].datatype[25].sstruct.field[23].datatype -389833101 -documenttype[0].datatype[25].sstruct.field[23].detailedtype "" -documenttype[0].datatype[25].sstruct.field[24].name "juletre" -documenttype[0].datatype[25].sstruct.field[24].id 1039981530 -documenttype[0].datatype[25].sstruct.field[24].datatype 4 -documenttype[0].datatype[25].sstruct.field[24].detailedtype "" -documenttype[0].datatype[25].sstruct.field[25].name "album0" -documenttype[0].datatype[25].sstruct.field[25].id 764312262 -documenttype[0].datatype[25].sstruct.field[25].datatype 18 -documenttype[0].datatype[25].sstruct.field[25].detailedtype "" -documenttype[0].datatype[25].sstruct.field[26].name "album1" -documenttype[0].datatype[25].sstruct.field[26].id 1967160809 -documenttype[0].datatype[25].sstruct.field[26].datatype 18 -documenttype[0].datatype[25].sstruct.field[26].detailedtype "" -documenttype[0].datatype[25].sstruct.field[27].name "other" -documenttype[0].datatype[25].sstruct.field[27].id 2443357 -documenttype[0].datatype[25].sstruct.field[27].datatype 4 -documenttype[0].datatype[25].sstruct.field[27].detailedtype "" -documenttype[0].fieldsets{[document]}.fields[0] "Folders" -documenttype[0].fieldsets{[document]}.fields[1] "abyte" -documenttype[0].fieldsets{[document]}.fields[2] "album0" -documenttype[0].fieldsets{[document]}.fields[3] "album1" -documenttype[0].fieldsets{[document]}.fields[4] "along" -documenttype[0].fieldsets{[document]}.fields[5] "arrarr" -documenttype[0].fieldsets{[document]}.fields[6] "arrayfield" -documenttype[0].fieldsets{[document]}.fields[7] "arraymapfield" -documenttype[0].fieldsets{[document]}.fields[8] "complexarray" -documenttype[0].fieldsets{[document]}.fields[9] "doublemapfield" -documenttype[0].fieldsets{[document]}.fields[10] "floatmapfield" -documenttype[0].fieldsets{[document]}.fields[11] "intmapfield" -documenttype[0].fieldsets{[document]}.fields[12] "juletre" -documenttype[0].fieldsets{[document]}.fields[13] "longmapfield" -documenttype[0].fieldsets{[document]}.fields[14] "maparr" -documenttype[0].fieldsets{[document]}.fields[15] "mystructarr" -documenttype[0].fieldsets{[document]}.fields[16] "mystructfield" -documenttype[0].fieldsets{[document]}.fields[17] "mystructmap" -documenttype[0].fieldsets{[document]}.fields[18] "pos" -documenttype[0].fieldsets{[document]}.fields[19] "setfield" -documenttype[0].fieldsets{[document]}.fields[20] "setfield2" -documenttype[0].fieldsets{[document]}.fields[21] "setfield3" -documenttype[0].fieldsets{[document]}.fields[22] "setfield4" -documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" -documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" -documenttype[0].fieldsets{[document]}.fields[25] "structfield" -documenttype[0].fieldsets{[document]}.fields[26] "tagfield" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "types" +doctype[1].idx 10015 +doctype[1].internalid -853072901 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "Folders" +doctype[1].fieldsets{[document]}.fields[1] "abyte" +doctype[1].fieldsets{[document]}.fields[2] "album0" +doctype[1].fieldsets{[document]}.fields[3] "album1" +doctype[1].fieldsets{[document]}.fields[4] "along" +doctype[1].fieldsets{[document]}.fields[5] "arrarr" +doctype[1].fieldsets{[document]}.fields[6] "arrayfield" +doctype[1].fieldsets{[document]}.fields[7] "arraymapfield" +doctype[1].fieldsets{[document]}.fields[8] "complexarray" +doctype[1].fieldsets{[document]}.fields[9] "doublemapfield" +doctype[1].fieldsets{[document]}.fields[10] "floatmapfield" +doctype[1].fieldsets{[document]}.fields[11] "intmapfield" +doctype[1].fieldsets{[document]}.fields[12] "juletre" +doctype[1].fieldsets{[document]}.fields[13] "longmapfield" +doctype[1].fieldsets{[document]}.fields[14] "maparr" +doctype[1].fieldsets{[document]}.fields[15] "mystructarr" +doctype[1].fieldsets{[document]}.fields[16] "mystructfield" +doctype[1].fieldsets{[document]}.fields[17] "mystructmap" +doctype[1].fieldsets{[document]}.fields[18] "pos" +doctype[1].fieldsets{[document]}.fields[19] "setfield" +doctype[1].fieldsets{[document]}.fields[20] "setfield2" +doctype[1].fieldsets{[document]}.fields[21] "setfield3" +doctype[1].fieldsets{[document]}.fields[22] "setfield4" +doctype[1].fieldsets{[document]}.fields[23] "stringmapfield" +doctype[1].fieldsets{[document]}.fields[24] "structarrayfield" +doctype[1].fieldsets{[document]}.fields[25] "structfield" +doctype[1].fieldsets{[document]}.fields[26] "tagfield" +doctype[1].arraytype[0].idx 10017 +doctype[1].arraytype[0].elementtype 10007 +doctype[1].arraytype[0].internalid -1245117006 +doctype[1].arraytype[1].idx 10024 +doctype[1].arraytype[1].elementtype 10023 +doctype[1].arraytype[1].internalid -1244829667 +doctype[1].arraytype[2].idx 10031 +doctype[1].arraytype[2].elementtype 10007 +doctype[1].arraytype[2].internalid -1245117006 +doctype[1].arraytype[3].idx 10032 +doctype[1].arraytype[3].elementtype 10033 +doctype[1].arraytype[3].internalid -794985308 +doctype[1].arraytype[4].idx 10033 +doctype[1].arraytype[4].elementtype 10034 +doctype[1].arraytype[4].internalid 1707615575 +doctype[1].arraytype[5].idx 10034 +doctype[1].arraytype[5].elementtype 10012 +doctype[1].arraytype[5].internalid -1486737430 +doctype[1].arraytype[6].idx 10035 +doctype[1].arraytype[6].elementtype 10036 +doctype[1].arraytype[6].internalid 69621385 +doctype[1].arraytype[7].idx 10037 +doctype[1].arraytype[7].elementtype 10038 +doctype[1].arraytype[7].internalid 1416345047 +doctype[1].arraytype[8].idx 10039 +doctype[1].arraytype[8].elementtype 10040 +doctype[1].arraytype[8].internalid 1707615575 +doctype[1].arraytype[9].idx 10040 +doctype[1].arraytype[9].elementtype 10012 +doctype[1].arraytype[9].internalid -1486737430 +doctype[1].arraytype[10].idx 10042 +doctype[1].arraytype[10].elementtype 10003 +doctype[1].arraytype[10].internalid 49942803 +doctype[1].arraytype[11].idx 10045 +doctype[1].arraytype[11].elementtype 10041 +doctype[1].arraytype[11].internalid 759956026 +doctype[1].maptype[0].idx 10025 +doctype[1].maptype[0].keytype 10012 +doctype[1].maptype[0].valuetype 10012 +doctype[1].maptype[0].internalid 339965458 +doctype[1].maptype[1].idx 10026 +doctype[1].maptype[1].keytype 10012 +doctype[1].maptype[1].valuetype 10007 +doctype[1].maptype[1].internalid -1584287606 +doctype[1].maptype[2].idx 10027 +doctype[1].maptype[2].keytype 10012 +doctype[1].maptype[2].valuetype 10005 +doctype[1].maptype[2].internalid 2125154557 +doctype[1].maptype[3].idx 10028 +doctype[1].maptype[3].keytype 10007 +doctype[1].maptype[3].valuetype 10008 +doctype[1].maptype[3].internalid -1715531035 +doctype[1].maptype[4].idx 10029 +doctype[1].maptype[4].keytype 10007 +doctype[1].maptype[4].valuetype 10004 +doctype[1].maptype[4].internalid 2138385264 +doctype[1].maptype[5].idx 10030 +doctype[1].maptype[5].keytype 10012 +doctype[1].maptype[5].valuetype 10031 +doctype[1].maptype[5].internalid 435886609 +doctype[1].maptype[6].idx 10036 +doctype[1].maptype[6].keytype 10012 +doctype[1].maptype[6].valuetype 10012 +doctype[1].maptype[6].internalid 339965458 +doctype[1].maptype[7].idx 10038 +doctype[1].maptype[7].keytype 10007 +doctype[1].maptype[7].valuetype 10039 +doctype[1].maptype[7].internalid -372512406 +doctype[1].maptype[8].idx 10043 +doctype[1].maptype[8].keytype 10012 +doctype[1].maptype[8].valuetype 10012 +doctype[1].maptype[8].internalid 339965458 +doctype[1].maptype[9].idx 10044 +doctype[1].maptype[9].keytype 10007 +doctype[1].maptype[9].valuetype 10041 +doctype[1].maptype[9].internalid 1901258752 +doctype[1].maptype[10].idx 10046 +doctype[1].maptype[10].keytype 10007 +doctype[1].maptype[10].valuetype 10047 +doctype[1].maptype[10].internalid -389833101 +doctype[1].maptype[11].idx 10048 +doctype[1].maptype[11].keytype 10012 +doctype[1].maptype[11].valuetype 10008 +doctype[1].maptype[11].internalid -1865479609 +doctype[1].wsettype[0].idx 10018 +doctype[1].wsettype[0].elementtype 10012 +doctype[1].wsettype[0].createifnonexistent false +doctype[1].wsettype[0].removeifzero false +doctype[1].wsettype[0].internalid 1328286588 +doctype[1].wsettype[1].idx 10019 +doctype[1].wsettype[1].elementtype 10012 +doctype[1].wsettype[1].createifnonexistent true +doctype[1].wsettype[1].removeifzero true +doctype[1].wsettype[1].internalid 18 +doctype[1].wsettype[2].idx 10020 +doctype[1].wsettype[2].elementtype 10012 +doctype[1].wsettype[2].createifnonexistent false +doctype[1].wsettype[2].removeifzero true +doctype[1].wsettype[2].internalid 2125328771 +doctype[1].wsettype[3].idx 10021 +doctype[1].wsettype[3].elementtype 10012 +doctype[1].wsettype[3].createifnonexistent true +doctype[1].wsettype[3].removeifzero false +doctype[1].wsettype[3].internalid 2065577986 +doctype[1].wsettype[4].idx 10022 +doctype[1].wsettype[4].elementtype 10012 +doctype[1].wsettype[4].createifnonexistent true +doctype[1].wsettype[4].removeifzero true +doctype[1].wsettype[4].internalid 18 +doctype[1].wsettype[5].idx 10049 +doctype[1].wsettype[5].elementtype 10012 +doctype[1].wsettype[5].createifnonexistent true +doctype[1].wsettype[5].removeifzero true +doctype[1].wsettype[5].internalid 18 +doctype[1].wsettype[6].idx 10050 +doctype[1].wsettype[6].elementtype 10012 +doctype[1].wsettype[6].createifnonexistent true +doctype[1].wsettype[6].removeifzero true +doctype[1].wsettype[6].internalid 18 +doctype[1].structtype[0].idx 10023 +doctype[1].structtype[0].name "sct" +doctype[1].structtype[0].field[0].name "s1" +doctype[1].structtype[0].field[0].internalid 2146820765 +doctype[1].structtype[0].field[0].type 10012 +doctype[1].structtype[0].field[1].name "s2" +doctype[1].structtype[0].field[1].internalid 45366795 +doctype[1].structtype[0].field[1].type 10012 +doctype[1].structtype[0].internalid 109267174 +doctype[1].structtype[1].idx 10041 +doctype[1].structtype[1].name "mystruct" +doctype[1].structtype[1].field[0].name "bytearr" +doctype[1].structtype[1].field[0].internalid 1079701754 +doctype[1].structtype[1].field[0].type 10042 +doctype[1].structtype[1].field[1].name "mymap" +doctype[1].structtype[1].field[1].internalid 1954178122 +doctype[1].structtype[1].field[1].type 10043 +doctype[1].structtype[1].field[2].name "title" +doctype[1].structtype[1].field[2].internalid 567626448 +doctype[1].structtype[1].field[2].type 10012 +doctype[1].structtype[1].field[3].name "structfield" +doctype[1].structtype[1].field[3].internalid 1726890940 +doctype[1].structtype[1].field[3].type 10012 +doctype[1].structtype[1].internalid -2092985853 +doctype[1].structtype[2].idx 10047 +doctype[1].structtype[2].name "folder" +doctype[1].structtype[2].field[0].name "Version" +doctype[1].structtype[2].field[0].internalid 64430502 +doctype[1].structtype[2].field[0].type 10007 +doctype[1].structtype[2].field[1].name "Name" +doctype[1].structtype[2].field[1].internalid 2002760220 +doctype[1].structtype[2].field[1].type 10012 +doctype[1].structtype[2].field[2].name "FlagsCounter" +doctype[1].structtype[2].field[2].internalid 1741227606 +doctype[1].structtype[2].field[2].type 10048 +doctype[1].structtype[2].field[3].name "anotherfolder" +doctype[1].structtype[2].field[3].internalid 1582421848 +doctype[1].structtype[2].field[3].type 10047 +doctype[1].structtype[2].internalid 294108848 +doctype[1].structtype[3].idx 10016 +doctype[1].structtype[3].name "types.header" +doctype[1].structtype[3].field[0].name "abyte" +doctype[1].structtype[3].field[0].internalid 110138156 +doctype[1].structtype[3].field[0].type 10003 +doctype[1].structtype[3].field[1].name "along" +doctype[1].structtype[3].field[1].internalid 1206464520 +doctype[1].structtype[3].field[1].type 10008 +doctype[1].structtype[3].field[2].name "arrayfield" +doctype[1].structtype[3].field[2].internalid 965790107 +doctype[1].structtype[3].field[2].type 10017 +doctype[1].structtype[3].field[3].name "setfield" +doctype[1].structtype[3].field[3].internalid 761581914 +doctype[1].structtype[3].field[3].type 10018 +doctype[1].structtype[3].field[4].name "pos" +doctype[1].structtype[3].field[4].internalid 1041567475 +doctype[1].structtype[3].field[4].type 10009 +doctype[1].structtype[3].field[5].name "setfield2" +doctype[1].structtype[3].field[5].internalid 1066659198 +doctype[1].structtype[3].field[5].type 10019 +doctype[1].structtype[3].field[6].name "setfield3" +doctype[1].structtype[3].field[6].internalid 1180155772 +doctype[1].structtype[3].field[6].type 10020 +doctype[1].structtype[3].field[7].name "setfield4" +doctype[1].structtype[3].field[7].internalid 1254131631 +doctype[1].structtype[3].field[7].type 10021 +doctype[1].structtype[3].field[8].name "tagfield" +doctype[1].structtype[3].field[8].internalid 1653562069 +doctype[1].structtype[3].field[8].type 10022 +doctype[1].structtype[3].field[9].name "structfield" +doctype[1].structtype[3].field[9].internalid 486207386 +doctype[1].structtype[3].field[9].type 10023 +doctype[1].structtype[3].field[10].name "structarrayfield" +doctype[1].structtype[3].field[10].internalid 335048518 +doctype[1].structtype[3].field[10].type 10024 +doctype[1].structtype[3].field[11].name "stringmapfield" +doctype[1].structtype[3].field[11].internalid 117465687 +doctype[1].structtype[3].field[11].type 10025 +doctype[1].structtype[3].field[12].name "intmapfield" +doctype[1].structtype[3].field[12].internalid 121004462 +doctype[1].structtype[3].field[12].type 10026 +doctype[1].structtype[3].field[13].name "floatmapfield" +doctype[1].structtype[3].field[13].internalid 1239120925 +doctype[1].structtype[3].field[13].type 10027 +doctype[1].structtype[3].field[14].name "longmapfield" +doctype[1].structtype[3].field[14].internalid 477718745 +doctype[1].structtype[3].field[14].type 10028 +doctype[1].structtype[3].field[15].name "doublemapfield" +doctype[1].structtype[3].field[15].internalid 877047192 +doctype[1].structtype[3].field[15].type 10029 +doctype[1].structtype[3].field[16].name "arraymapfield" +doctype[1].structtype[3].field[16].internalid 1670805928 +doctype[1].structtype[3].field[16].type 10030 +doctype[1].structtype[3].field[17].name "arrarr" +doctype[1].structtype[3].field[17].internalid 1962567166 +doctype[1].structtype[3].field[17].type 10032 +doctype[1].structtype[3].field[18].name "maparr" +doctype[1].structtype[3].field[18].internalid 904375219 +doctype[1].structtype[3].field[18].type 10035 +doctype[1].structtype[3].field[19].name "complexarray" +doctype[1].structtype[3].field[19].internalid 795629533 +doctype[1].structtype[3].field[19].type 10037 +doctype[1].structtype[3].field[20].name "mystructfield" +doctype[1].structtype[3].field[20].internalid 1348513378 +doctype[1].structtype[3].field[20].type 10041 +doctype[1].structtype[3].field[21].name "mystructmap" +doctype[1].structtype[3].field[21].internalid 1511423250 +doctype[1].structtype[3].field[21].type 10044 +doctype[1].structtype[3].field[22].name "mystructarr" +doctype[1].structtype[3].field[22].internalid 595856991 +doctype[1].structtype[3].field[22].type 10045 +doctype[1].structtype[3].field[23].name "Folders" +doctype[1].structtype[3].field[23].internalid 34575524 +doctype[1].structtype[3].field[23].type 10046 +doctype[1].structtype[3].field[24].name "juletre" +doctype[1].structtype[3].field[24].internalid 1039981530 +doctype[1].structtype[3].field[24].type 10008 +doctype[1].structtype[3].field[25].name "album0" +doctype[1].structtype[3].field[25].internalid 764312262 +doctype[1].structtype[3].field[25].type 10049 +doctype[1].structtype[3].field[26].name "album1" +doctype[1].structtype[3].field[26].internalid 1967160809 +doctype[1].structtype[3].field[26].type 10050 +doctype[1].structtype[3].field[27].name "other" +doctype[1].structtype[3].field[27].internalid 2443357 +doctype[1].structtype[3].field[27].type 10008 +doctype[1].structtype[3].internalid 1328581348 diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index 61c92eee8d1..15430101553 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -1,49 +1,65 @@ enablecompression false usev8geopositions false -documenttype[0].id -1368624373 -documenttype[0].name "other_doc" -documenttype[0].version 0 -documenttype[0].headerstruct 1631005140 -documenttype[0].bodystruct 0 -documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id 1631005140 -documenttype[0].datatype[0].type STRUCT -documenttype[0].datatype[0].array.element.id 0 -documenttype[0].datatype[0].map.key.id 0 -documenttype[0].datatype[0].map.value.id 0 -documenttype[0].datatype[0].wset.key.id 0 -documenttype[0].datatype[0].wset.createifnonexistent false -documenttype[0].datatype[0].wset.removeifzero false -documenttype[0].datatype[0].annotationref.annotation.id 0 -documenttype[0].datatype[0].sstruct.name "other_doc.header" -documenttype[0].datatype[0].sstruct.version 0 -documenttype[0].datatype[0].sstruct.compression.type NONE -documenttype[0].datatype[0].sstruct.compression.level 0 -documenttype[0].datatype[0].sstruct.compression.threshold 95 -documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[1].id -853072901 -documenttype[1].name "types" -documenttype[1].version 0 -documenttype[1].headerstruct 1328581348 -documenttype[1].bodystruct 0 -documenttype[1].inherits[0].id 8 -documenttype[1].datatype[0].id 1328581348 -documenttype[1].datatype[0].type STRUCT -documenttype[1].datatype[0].array.element.id 0 -documenttype[1].datatype[0].map.key.id 0 -documenttype[1].datatype[0].map.value.id 0 -documenttype[1].datatype[0].wset.key.id 0 -documenttype[1].datatype[0].wset.createifnonexistent false -documenttype[1].datatype[0].wset.removeifzero false -documenttype[1].datatype[0].annotationref.annotation.id 0 -documenttype[1].datatype[0].sstruct.name "types.header" -documenttype[1].datatype[0].sstruct.version 0 -documenttype[1].datatype[0].sstruct.compression.type NONE -documenttype[1].datatype[0].sstruct.compression.level 0 -documenttype[1].datatype[0].sstruct.compression.threshold 95 -documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[0].sstruct.field[0].name "doc_field" -documenttype[1].datatype[0].sstruct.field[0].id 819293364 -documenttype[1].datatype[0].sstruct.field[0].datatype -1368624373 -documenttype[1].datatype[0].sstruct.field[0].detailedtype "" -documenttype[1].fieldsets{[document]}.fields[0] "doc_field" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "other_doc" +doctype[1].idx 10015 +doctype[1].internalid -1368624373 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "other_doc.header" +doctype[1].structtype[0].internalid 1631005140 +doctype[2].name "types" +doctype[2].idx 10017 +doctype[2].internalid -853072901 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "doc_field" +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "types.header" +doctype[2].structtype[0].field[0].name "doc_field" +doctype[2].structtype[0].field[0].internalid 819293364 +doctype[2].structtype[0].field[0].type 10015 +doctype[2].structtype[0].internalid 1328581348 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index d992839d5d9..1b5817e6f39 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -1,97 +1,98 @@ enablecompression false usev8geopositions false -documenttype[0].id 2987301 -documenttype[0].name "ad" -documenttype[0].version 0 -documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct 0 -documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id 959075962 -documenttype[0].datatype[0].type STRUCT -documenttype[0].datatype[0].array.element.id 0 -documenttype[0].datatype[0].map.key.id 0 -documenttype[0].datatype[0].map.value.id 0 -documenttype[0].datatype[0].wset.key.id 0 -documenttype[0].datatype[0].wset.createifnonexistent false -documenttype[0].datatype[0].wset.removeifzero false -documenttype[0].datatype[0].annotationref.annotation.id 0 -documenttype[0].datatype[0].sstruct.name "ad.header" -documenttype[0].datatype[0].sstruct.version 0 -documenttype[0].datatype[0].sstruct.compression.type NONE -documenttype[0].datatype[0].sstruct.compression.level 0 -documenttype[0].datatype[0].sstruct.compression.threshold 95 -documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" -documenttype[0].datatype[0].sstruct.field[0].id 23963250 -documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 -documenttype[0].datatype[0].sstruct.field[0].detailedtype "" -documenttype[0].datatype[0].sstruct.field[1].name "person_ref" -documenttype[0].datatype[0].sstruct.field[1].id 100779805 -documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 -documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -documenttype[0].referencetype[0].id 595216861 -documenttype[0].referencetype[0].target_type_id -1318255918 -documenttype[0].referencetype[1].id 542332920 -documenttype[0].referencetype[1].target_type_id 443162583 -documenttype[0].importedfield[0].name "my_cool_field" -documenttype[0].importedfield[1].name "my_swag_field" -documenttype[0].importedfield[2].name "my_name" -documenttype[1].id -1318255918 -documenttype[1].name "campaign" -documenttype[1].version 0 -documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 0 -documenttype[1].inherits[0].id 8 -documenttype[1].datatype[0].id -2041471955 -documenttype[1].datatype[0].type STRUCT -documenttype[1].datatype[0].array.element.id 0 -documenttype[1].datatype[0].map.key.id 0 -documenttype[1].datatype[0].map.value.id 0 -documenttype[1].datatype[0].wset.key.id 0 -documenttype[1].datatype[0].wset.createifnonexistent false -documenttype[1].datatype[0].wset.removeifzero false -documenttype[1].datatype[0].annotationref.annotation.id 0 -documenttype[1].datatype[0].sstruct.name "campaign.header" -documenttype[1].datatype[0].sstruct.version 0 -documenttype[1].datatype[0].sstruct.compression.type NONE -documenttype[1].datatype[0].sstruct.compression.level 0 -documenttype[1].datatype[0].sstruct.compression.threshold 95 -documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[0].sstruct.field[0].name "cool_field" -documenttype[1].datatype[0].sstruct.field[0].id 1588702436 -documenttype[1].datatype[0].sstruct.field[0].datatype 2 -documenttype[1].datatype[0].sstruct.field[0].detailedtype "" -documenttype[1].datatype[0].sstruct.field[1].name "swag_field" -documenttype[1].datatype[0].sstruct.field[1].id 1691224741 -documenttype[1].datatype[0].sstruct.field[1].datatype 4 -documenttype[1].datatype[0].sstruct.field[1].detailedtype "" -documenttype[1].fieldsets{[document]}.fields[0] "cool_field" -documenttype[1].fieldsets{[document]}.fields[1] "swag_field" -documenttype[2].id 443162583 -documenttype[2].name "person" -documenttype[2].version 0 -documenttype[2].headerstruct 3129224 -documenttype[2].bodystruct 0 -documenttype[2].inherits[0].id 8 -documenttype[2].datatype[0].id 3129224 -documenttype[2].datatype[0].type STRUCT -documenttype[2].datatype[0].array.element.id 0 -documenttype[2].datatype[0].map.key.id 0 -documenttype[2].datatype[0].map.value.id 0 -documenttype[2].datatype[0].wset.key.id 0 -documenttype[2].datatype[0].wset.createifnonexistent false -documenttype[2].datatype[0].wset.removeifzero false -documenttype[2].datatype[0].annotationref.annotation.id 0 -documenttype[2].datatype[0].sstruct.name "person.header" -documenttype[2].datatype[0].sstruct.version 0 -documenttype[2].datatype[0].sstruct.compression.type NONE -documenttype[2].datatype[0].sstruct.compression.level 0 -documenttype[2].datatype[0].sstruct.compression.threshold 95 -documenttype[2].datatype[0].sstruct.compression.minsize 200 -documenttype[2].datatype[0].sstruct.field[0].name "name" -documenttype[2].datatype[0].sstruct.field[0].id 1160796772 -documenttype[2].datatype[0].sstruct.field[0].datatype 2 -documenttype[2].datatype[0].sstruct.field[0].detailedtype "" -documenttype[2].fieldsets{[document]}.fields[0] "name" +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "ad" +doctype[1].idx 10015 +doctype[1].internalid 2987301 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[1].fieldsets{[document]}.fields[1] "person_ref" +doctype[1].importedfield[0].name "my_cool_field" +doctype[1].importedfield[1].name "my_swag_field" +doctype[1].importedfield[2].name "my_name" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[0].internalid 595216861 +doctype[1].documentref[1].idx 10019 +doctype[1].documentref[1].targettype 10020 +doctype[1].documentref[1].internalid 542332920 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "ad.header" +doctype[1].structtype[0].field[0].name "campaign_ref" +doctype[1].structtype[0].field[0].internalid 23963250 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "person_ref" +doctype[1].structtype[0].field[1].internalid 100779805 +doctype[1].structtype[0].field[1].type 10019 +doctype[1].structtype[0].internalid 959075962 +doctype[2].name "campaign" +doctype[2].idx 10018 +doctype[2].internalid -1318255918 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10021 +doctype[2].fieldsets{[document]}.fields[0] "cool_field" +doctype[2].fieldsets{[document]}.fields[1] "swag_field" +doctype[2].structtype[0].idx 10021 +doctype[2].structtype[0].name "campaign.header" +doctype[2].structtype[0].field[0].name "cool_field" +doctype[2].structtype[0].field[0].internalid 1588702436 +doctype[2].structtype[0].field[0].type 10012 +doctype[2].structtype[0].field[1].name "swag_field" +doctype[2].structtype[0].field[1].internalid 1691224741 +doctype[2].structtype[0].field[1].type 10008 +doctype[2].structtype[0].internalid -2041471955 +doctype[3].name "person" +doctype[3].idx 10020 +doctype[3].internalid 443162583 +doctype[3].inherits[0].idx 10000 +doctype[3].contentstruct 10022 +doctype[3].fieldsets{[document]}.fields[0] "name" +doctype[3].structtype[0].idx 10022 +doctype[3].structtype[0].name "person.header" +doctype[3].structtype[0].field[0].name "name" +doctype[3].structtype[0].field[0].internalid 1160796772 +doctype[3].structtype[0].field[0].type 10012 +doctype[3].structtype[0].internalid 3129224 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index 68ed924615f..1c5d4d41819 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -1,79 +1,83 @@ enablecompression false usev8geopositions false -documenttype[0].id 2987301 -documenttype[0].name "ad" -documenttype[0].version 0 -documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct 0 -documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id 959075962 -documenttype[0].datatype[0].type STRUCT -documenttype[0].datatype[0].array.element.id 0 -documenttype[0].datatype[0].map.key.id 0 -documenttype[0].datatype[0].map.value.id 0 -documenttype[0].datatype[0].wset.key.id 0 -documenttype[0].datatype[0].wset.createifnonexistent false -documenttype[0].datatype[0].wset.removeifzero false -documenttype[0].datatype[0].annotationref.annotation.id 0 -documenttype[0].datatype[0].sstruct.name "ad.header" -documenttype[0].datatype[0].sstruct.version 0 -documenttype[0].datatype[0].sstruct.compression.type NONE -documenttype[0].datatype[0].sstruct.compression.level 0 -documenttype[0].datatype[0].sstruct.compression.threshold 95 -documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" -documenttype[0].datatype[0].sstruct.field[0].id 23963250 -documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 -documenttype[0].datatype[0].sstruct.field[0].detailedtype "" -documenttype[0].datatype[0].sstruct.field[1].name "person_ref" -documenttype[0].datatype[0].sstruct.field[1].id 100779805 -documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 -documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -documenttype[0].referencetype[0].id 595216861 -documenttype[0].referencetype[0].target_type_id -1318255918 -documenttype[0].referencetype[1].id 542332920 -documenttype[0].referencetype[1].target_type_id 443162583 -documenttype[1].id -1318255918 -documenttype[1].name "campaign" -documenttype[1].version 0 -documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 0 -documenttype[1].inherits[0].id 8 -documenttype[1].datatype[0].id -2041471955 -documenttype[1].datatype[0].type STRUCT -documenttype[1].datatype[0].array.element.id 0 -documenttype[1].datatype[0].map.key.id 0 -documenttype[1].datatype[0].map.value.id 0 -documenttype[1].datatype[0].wset.key.id 0 -documenttype[1].datatype[0].wset.createifnonexistent false -documenttype[1].datatype[0].wset.removeifzero false -documenttype[1].datatype[0].annotationref.annotation.id 0 -documenttype[1].datatype[0].sstruct.name "campaign.header" -documenttype[1].datatype[0].sstruct.version 0 -documenttype[1].datatype[0].sstruct.compression.type NONE -documenttype[1].datatype[0].sstruct.compression.level 0 -documenttype[1].datatype[0].sstruct.compression.threshold 95 -documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[2].id 443162583 -documenttype[2].name "person" -documenttype[2].version 0 -documenttype[2].headerstruct 3129224 -documenttype[2].bodystruct 0 -documenttype[2].inherits[0].id 8 -documenttype[2].datatype[0].id 3129224 -documenttype[2].datatype[0].type STRUCT -documenttype[2].datatype[0].array.element.id 0 -documenttype[2].datatype[0].map.key.id 0 -documenttype[2].datatype[0].map.value.id 0 -documenttype[2].datatype[0].wset.key.id 0 -documenttype[2].datatype[0].wset.createifnonexistent false -documenttype[2].datatype[0].wset.removeifzero false -documenttype[2].datatype[0].annotationref.annotation.id 0 -documenttype[2].datatype[0].sstruct.name "person.header" -documenttype[2].datatype[0].sstruct.version 0 -documenttype[2].datatype[0].sstruct.compression.type NONE -documenttype[2].datatype[0].sstruct.compression.level 0 -documenttype[2].datatype[0].sstruct.compression.threshold 95 -documenttype[2].datatype[0].sstruct.compression.minsize 200 +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "ad" +doctype[1].idx 10015 +doctype[1].internalid 2987301 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[1].fieldsets{[document]}.fields[1] "person_ref" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[0].internalid 595216861 +doctype[1].documentref[1].idx 10019 +doctype[1].documentref[1].targettype 10020 +doctype[1].documentref[1].internalid 542332920 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "ad.header" +doctype[1].structtype[0].field[0].name "campaign_ref" +doctype[1].structtype[0].field[0].internalid 23963250 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "person_ref" +doctype[1].structtype[0].field[1].internalid 100779805 +doctype[1].structtype[0].field[1].type 10019 +doctype[1].structtype[0].internalid 959075962 +doctype[2].name "campaign" +doctype[2].idx 10018 +doctype[2].internalid -1318255918 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10021 +doctype[2].structtype[0].idx 10021 +doctype[2].structtype[0].name "campaign.header" +doctype[2].structtype[0].internalid -2041471955 +doctype[3].name "person" +doctype[3].idx 10020 +doctype[3].internalid 443162583 +doctype[3].inherits[0].idx 10000 +doctype[3].contentstruct 10022 +doctype[3].structtype[0].idx 10022 +doctype[3].structtype[0].name "person.header" +doctype[3].structtype[0].internalid 3129224 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index 6415e62cd7e..2f178c55bfd 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -1,56 +1,72 @@ enablecompression false usev8geopositions false -documenttype[0].id 2987301 -documenttype[0].name "ad" -documenttype[0].version 0 -documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct 0 -documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id 959075962 -documenttype[0].datatype[0].type STRUCT -documenttype[0].datatype[0].array.element.id 0 -documenttype[0].datatype[0].map.key.id 0 -documenttype[0].datatype[0].map.value.id 0 -documenttype[0].datatype[0].wset.key.id 0 -documenttype[0].datatype[0].wset.createifnonexistent false -documenttype[0].datatype[0].wset.removeifzero false -documenttype[0].datatype[0].annotationref.annotation.id 0 -documenttype[0].datatype[0].sstruct.name "ad.header" -documenttype[0].datatype[0].sstruct.version 0 -documenttype[0].datatype[0].sstruct.compression.type NONE -documenttype[0].datatype[0].sstruct.compression.level 0 -documenttype[0].datatype[0].sstruct.compression.threshold 95 -documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" -documenttype[0].datatype[0].sstruct.field[0].id 23963250 -documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 -documenttype[0].datatype[0].sstruct.field[0].detailedtype "" -documenttype[0].datatype[0].sstruct.field[1].name "other_campaign_ref" -documenttype[0].datatype[0].sstruct.field[1].id 874751172 -documenttype[0].datatype[0].sstruct.field[1].datatype 595216861 -documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" -documenttype[0].referencetype[0].id 595216861 -documenttype[0].referencetype[0].target_type_id -1318255918 -documenttype[1].id -1318255918 -documenttype[1].name "campaign" -documenttype[1].version 0 -documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 0 -documenttype[1].inherits[0].id 8 -documenttype[1].datatype[0].id -2041471955 -documenttype[1].datatype[0].type STRUCT -documenttype[1].datatype[0].array.element.id 0 -documenttype[1].datatype[0].map.key.id 0 -documenttype[1].datatype[0].map.value.id 0 -documenttype[1].datatype[0].wset.key.id 0 -documenttype[1].datatype[0].wset.createifnonexistent false -documenttype[1].datatype[0].wset.removeifzero false -documenttype[1].datatype[0].annotationref.annotation.id 0 -documenttype[1].datatype[0].sstruct.name "campaign.header" -documenttype[1].datatype[0].sstruct.version 0 -documenttype[1].datatype[0].sstruct.compression.type NONE -documenttype[1].datatype[0].sstruct.compression.level 0 -documenttype[1].datatype[0].sstruct.compression.threshold 95 -documenttype[1].datatype[0].sstruct.compression.minsize 200 +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "ad" +doctype[1].idx 10015 +doctype[1].internalid 2987301 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[0].internalid 595216861 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "ad.header" +doctype[1].structtype[0].field[0].name "campaign_ref" +doctype[1].structtype[0].field[0].internalid 23963250 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "other_campaign_ref" +doctype[1].structtype[0].field[1].internalid 874751172 +doctype[1].structtype[0].field[1].type 10017 +doctype[1].structtype[0].internalid 959075962 +doctype[2].name "campaign" +doctype[2].idx 10018 +doctype[2].internalid -1318255918 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10019 +doctype[2].structtype[0].idx 10019 +doctype[2].structtype[0].name "campaign.header" +doctype[2].structtype[0].internalid -2041471955 diff --git a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg index 1b897214d73..867e1e70e7a 100644 --- a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg +++ b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg @@ -1,100 +1,88 @@ enablecompression false usev8geopositions false -documenttype[].id 97614088 -documenttype[].name "foo" -documenttype[].version 0 -documenttype[].headerstruct -308552393 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id -2092985853 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "mystruct" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "name" -documenttype[].datatype[].sstruct.field[].id 1160796772 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "company" -documenttype[].datatype[].sstruct.field[].id 2010814026 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id 759956026 -documenttype[].datatype[].type ARRAY -documenttype[].datatype[].array.element.id -2092985853 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].id -308552393 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "foo.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "timestamp" -documenttype[].datatype[].sstruct.field[].id 808678733 -documenttype[].datatype[].sstruct.field[].datatype 4 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "mystuff" -documenttype[].datatype[].sstruct.field[].id 885106505 -documenttype[].datatype[].sstruct.field[].datatype 759956026 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "mystuff" -documenttype[].fieldsets{[document]}.fields[] "timestamp" -documenttype[].id 378030095 -documenttype[].name "foobar" -documenttype[].version 0 -documenttype[].headerstruct -1365874608 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].inherits[].id 97614088 -documenttype[].datatype[].id -1365874608 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "foobar.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "title" -documenttype[].datatype[].sstruct.field[].id 567626448 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "mystuff" -documenttype[].fieldsets{[document]}.fields[] "timestamp" -documenttype[].fieldsets{[document]}.fields[] "title" +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "foo" +doctype[].idx 10015 +doctype[].internalid 97614088 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "mystuff" +doctype[].fieldsets{[document]}.fields[] "timestamp" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10018 +doctype[].arraytype[].internalid 759956026 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "mystruct" +doctype[].structtype[].field[].name "name" +doctype[].structtype[].field[].internalid 1160796772 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "company" +doctype[].structtype[].field[].internalid 2010814026 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid -2092985853 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "foo.header" +doctype[].structtype[].field[].name "timestamp" +doctype[].structtype[].field[].internalid 808678733 +doctype[].structtype[].field[].type 10008 +doctype[].structtype[].field[].name "mystuff" +doctype[].structtype[].field[].internalid 885106505 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].internalid -308552393 +doctype[].name "foobar" +doctype[].idx 10019 +doctype[].internalid 378030095 +doctype[].inherits[].idx 10000 +doctype[].inherits[].idx 10015 +doctype[].contentstruct 10020 +doctype[].fieldsets{[document]}.fields[] "mystuff" +doctype[].fieldsets{[document]}.fields[] "timestamp" +doctype[].fieldsets{[document]}.fields[] "title" +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "foobar.header" +doctype[].structtype[].field[].name "title" +doctype[].structtype[].field[].internalid 567626448 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid -1365874608 diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg index c3ea1318d33..7a67640adfe 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -1,110 +1,99 @@ enablecompression false usev8geopositions false -documenttype[].id -94853056 -documenttype[].name "child_a" -documenttype[].version 0 -documenttype[].headerstruct 867409663 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id 867409663 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child_a.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "ref_from_a" -documenttype[].datatype[].sstruct.field[].id 300427062 -documenttype[].datatype[].sstruct.field[].datatype 427398467 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "ref_from_a" -documenttype[].referencetype[].id 427398467 -documenttype[].referencetype[].target_type_id 1175161836 -documenttype[].id -94852095 -documenttype[].name "child_b" -documenttype[].version 0 -documenttype[].headerstruct 670896158 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].inherits[].id -94853056 -documenttype[].datatype[].id 670896158 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child_b.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "ref_from_b" -documenttype[].datatype[].sstruct.field[].id 185778735 -documenttype[].datatype[].sstruct.field[].datatype 427398467 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "ref_from_a" -documenttype[].fieldsets{[document]}.fields[] "ref_from_b" -documenttype[].id -94851134 -documenttype[].name "child_c" -documenttype[].version 0 -documenttype[].headerstruct 474382653 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].inherits[].id -94852095 -documenttype[].datatype[].id 474382653 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child_c.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].fieldsets{[document]}.fields[] "ref_from_a" -documenttype[].fieldsets{[document]}.fields[] "ref_from_b" -documenttype[].importedfield[].name "from_a_int_field" -documenttype[].importedfield[].name "from_b_int_field" -documenttype[].id 1175161836 -documenttype[].name "parent" -documenttype[].version 0 -documenttype[].headerstruct 836075987 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id 836075987 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "parent.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "int_field" -documenttype[].datatype[].sstruct.field[].id 2128822283 -documenttype[].datatype[].sstruct.field[].datatype 0 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "int_field" +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "child_a" +doctype[].idx 10015 +doctype[].internalid -94853056 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "ref_from_a" +doctype[].documentref[].idx 10017 +doctype[].documentref[].targettype 10018 +doctype[].documentref[].internalid 427398467 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "child_a.header" +doctype[].structtype[].field[].name "ref_from_a" +doctype[].structtype[].field[].internalid 300427062 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].internalid 867409663 +doctype[].name "child_b" +doctype[].idx 10019 +doctype[].internalid -94852095 +doctype[].inherits[].idx 10000 +doctype[].inherits[].idx 10015 +doctype[].contentstruct 10020 +doctype[].fieldsets{[document]}.fields[] "ref_from_a" +doctype[].fieldsets{[document]}.fields[] "ref_from_b" +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "child_b.header" +doctype[].structtype[].field[].name "ref_from_b" +doctype[].structtype[].field[].internalid 185778735 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].internalid 670896158 +doctype[].name "child_c" +doctype[].idx 10021 +doctype[].internalid -94851134 +doctype[].inherits[].idx 10000 +doctype[].inherits[].idx 10019 +doctype[].contentstruct 10022 +doctype[].fieldsets{[document]}.fields[] "ref_from_a" +doctype[].fieldsets{[document]}.fields[] "ref_from_b" +doctype[].importedfield[].name "from_a_int_field" +doctype[].importedfield[].name "from_b_int_field" +doctype[].structtype[].idx 10022 +doctype[].structtype[].name "child_c.header" +doctype[].structtype[].internalid 474382653 +doctype[].name "parent" +doctype[].idx 10018 +doctype[].internalid 1175161836 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10023 +doctype[].fieldsets{[document]}.fields[] "int_field" +doctype[].structtype[].idx 10023 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "int_field" +doctype[].structtype[].field[].internalid 2128822283 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 836075987 diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index da65510ee5a..e8fd97671ff 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -1,79 +1,80 @@ enablecompression false usev8geopositions false -documenttype[].id 1175161836 -documenttype[].name "parent" -documenttype[].version 0 -documenttype[].headerstruct 836075987 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id 836075987 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "parent.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "weight_src" -documenttype[].datatype[].sstruct.field[].id 1225660233 -documenttype[].datatype[].sstruct.field[].datatype 1 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "weight" -documenttype[].datatype[].sstruct.field[].id 1001392207 -documenttype[].datatype[].sstruct.field[].datatype 1 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id 1091188812 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "parent_struct" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "parent_field" -documenttype[].datatype[].sstruct.field[].id 933533022 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "weight_src" -documenttype[].id 746267614 -documenttype[].name "child" -documenttype[].version 0 -documenttype[].headerstruct 81425825 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].inherits[].id 1175161836 -documenttype[].datatype[].id 81425825 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "child_field" -documenttype[].datatype[].sstruct.field[].id 1814271363 -documenttype[].datatype[].sstruct.field[].datatype 1091188812 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "child_field" -documenttype[].fieldsets{[document]}.fields[] "weight_src" +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "parent" +doctype[].idx 10015 +doctype[].internalid 1175161836 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "weight_src" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "weight_src" +doctype[].structtype[].field[].internalid 1225660233 +doctype[].structtype[].field[].type 10005 +doctype[].structtype[].field[].name "weight" +doctype[].structtype[].field[].internalid 1001392207 +doctype[].structtype[].field[].type 10005 +doctype[].structtype[].internalid 836075987 +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "parent_struct" +doctype[].structtype[].field[].name "parent_field" +doctype[].structtype[].field[].internalid 933533022 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid 1091188812 +doctype[].name "child" +doctype[].idx 10018 +doctype[].internalid 746267614 +doctype[].inherits[].idx 10000 +doctype[].inherits[].idx 10015 +doctype[].contentstruct 10019 +doctype[].fieldsets{[document]}.fields[] "child_field" +doctype[].fieldsets{[document]}.fields[] "weight_src" +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "child.header" +doctype[].structtype[].field[].name "child_field" +doctype[].structtype[].field[].internalid 1814271363 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].internalid 81425825 diff --git a/config-model/src/test/derived/multi_struct/documenttypes.cfg b/config-model/src/test/derived/multi_struct/documenttypes.cfg index c358b2854ee..9ce19079177 100644 --- a/config-model/src/test/derived/multi_struct/documenttypes.cfg +++ b/config-model/src/test/derived/multi_struct/documenttypes.cfg @@ -1,194 +1,128 @@ enablecompression false usev8geopositions false -documenttype[].id 2987301 -documenttype[].name "ad" -documenttype[].version 0 -documenttype[].headerstruct 959075962 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id 959075962 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "ad.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "e" -documenttype[].datatype[].sstruct.field[].id 970377814 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "e" -documenttype[].id -1051831567 -documenttype[].name "product" -documenttype[].version 0 -documenttype[].headerstruct -107300050 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id -2092985853 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "mystruct" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "zero" -documenttype[].datatype[].sstruct.field[].id 2128579715 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "one" -documenttype[].datatype[].sstruct.field[].id 997119011 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "two" -documenttype[].datatype[].sstruct.field[].id 2054688289 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "three" -documenttype[].datatype[].sstruct.field[].id 814368361 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -420192670 -documenttype[].datatype[].type MAP -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 2 -documenttype[].datatype[].map.value.id -2092985853 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].id -107300050 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "product.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "more_stuff" -documenttype[].datatype[].sstruct.field[].id 278342855 -documenttype[].datatype[].sstruct.field[].datatype -420192670 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "more_stuff" -documenttype[].id -903152840 -documenttype[].name "shop" -documenttype[].version 0 -documenttype[].headerstruct 371492103 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id -2092985853 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "mystruct" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "one" -documenttype[].datatype[].sstruct.field[].id 997119011 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "two" -documenttype[].datatype[].sstruct.field[].id 2054688289 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "three" -documenttype[].datatype[].sstruct.field[].id 814368361 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -420192670 -documenttype[].datatype[].type MAP -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 2 -documenttype[].datatype[].map.value.id -2092985853 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].id 371492103 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "shop.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "some_stuff" -documenttype[].datatype[].sstruct.field[].id 1543312381 -documenttype[].datatype[].sstruct.field[].datatype -420192670 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "some_stuff" -documenttype[].id -836031795 -documenttype[].name "user" -documenttype[].version 0 -documenttype[].headerstruct 1601213394 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id 1601213394 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "user.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "a" -documenttype[].datatype[].sstruct.field[].id 493339625 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "a" +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "ad" +doctype[].idx 10015 +doctype[].internalid 2987301 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "e" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "ad.header" +doctype[].structtype[].field[].name "e" +doctype[].structtype[].field[].internalid 970377814 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid 959075962 +doctype[].name "product" +doctype[].idx 10017 +doctype[].internalid -1051831567 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10018 +doctype[].fieldsets{[document]}.fields[] "more_stuff" +doctype[].maptype[].idx 10019 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10020 +doctype[].maptype[].internalid -420192670 +doctype[].structtype[].idx 10020 +doctype[].structtype[].name "mystruct" +doctype[].structtype[].field[].name "zero" +doctype[].structtype[].field[].internalid 2128579715 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "one" +doctype[].structtype[].field[].internalid 997119011 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "two" +doctype[].structtype[].field[].internalid 2054688289 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "three" +doctype[].structtype[].field[].internalid 814368361 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid -2092985853 +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "product.header" +doctype[].structtype[].field[].name "more_stuff" +doctype[].structtype[].field[].internalid 278342855 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].internalid -107300050 +doctype[].name "shop" +doctype[].idx 10021 +doctype[].internalid -903152840 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10022 +doctype[].fieldsets{[document]}.fields[] "some_stuff" +doctype[].maptype[].idx 10023 +doctype[].maptype[].keytype 10012 +doctype[].maptype[].valuetype 10024 +doctype[].maptype[].internalid -420192670 +doctype[].structtype[].idx 10024 +doctype[].structtype[].name "mystruct" +doctype[].structtype[].field[].name "one" +doctype[].structtype[].field[].internalid 997119011 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "two" +doctype[].structtype[].field[].internalid 2054688289 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "three" +doctype[].structtype[].field[].internalid 814368361 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid -2092985853 +doctype[].structtype[].idx 10022 +doctype[].structtype[].name "shop.header" +doctype[].structtype[].field[].name "some_stuff" +doctype[].structtype[].field[].internalid 1543312381 +doctype[].structtype[].field[].type 10023 +doctype[].structtype[].internalid 371492103 +doctype[].name "user" +doctype[].idx 10025 +doctype[].internalid -836031795 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10026 +doctype[].fieldsets{[document]}.fields[] "a" +doctype[].structtype[].idx 10026 +doctype[].structtype[].name "user.header" +doctype[].structtype[].field[].name "a" +doctype[].structtype[].field[].internalid 493339625 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid 1601213394 diff --git a/config-model/src/test/derived/structandfieldset/attributes.cfg b/config-model/src/test/derived/structandfieldset/attributes.cfg new file mode 100644 index 00000000000..b441857c9c7 --- /dev/null +++ b/config-model/src/test/derived/structandfieldset/attributes.cfg @@ -0,0 +1,96 @@ +attribute[].name "tag" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "people.first_name" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "people.last_name" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].dictionary.type BTREE +attribute[].dictionary.match UNCASED +attribute[].match UNCASED +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].paged false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors true +attribute[].enableonlybitvector true +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].maxuncommittedmemory 77777 +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/structandfieldset/test.sd b/config-model/src/test/derived/structandfieldset/test.sd index 77316eab9c9..da12787e564 100644 --- a/config-model/src/test/derived/structandfieldset/test.sd +++ b/config-model/src/test/derived/structandfieldset/test.sd @@ -14,7 +14,10 @@ schema test { field people type array<person> { indexing: summary struct-field first_name { indexing: attribute } - struct-field last_name { indexing: attribute } + struct-field last_name { + indexing: attribute + rank: filter + } } } diff --git a/config-model/src/test/derived/structinheritance/documenttypes.cfg b/config-model/src/test/derived/structinheritance/documenttypes.cfg index cf4bc89866f..8b343665289 100644 --- a/config-model/src/test/derived/structinheritance/documenttypes.cfg +++ b/config-model/src/test/derived/structinheritance/documenttypes.cfg @@ -1,102 +1,81 @@ enablecompression false usev8geopositions false -documenttype[].id 485659380 -documenttype[].name "simple" -documenttype[].version 0 -documenttype[].headerstruct -2142109237 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id -1396204461 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "base" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "name" -documenttype[].datatype[].sstruct.field[].id 1160796772 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id 746267614 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "age" -documenttype[].datatype[].sstruct.field[].id 1862473705 -documenttype[].datatype[].sstruct.field[].datatype 0 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "name" -documenttype[].datatype[].sstruct.field[].id 1160796772 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id 1811766610 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "grandchild" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "toy" -documenttype[].datatype[].sstruct.field[].id 536645790 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "age" -documenttype[].datatype[].sstruct.field[].id 1862473705 -documenttype[].datatype[].sstruct.field[].datatype 0 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "name" -documenttype[].datatype[].sstruct.field[].id 1160796772 -documenttype[].datatype[].sstruct.field[].datatype 2 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -2142109237 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "simple.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "f1" -documenttype[].datatype[].sstruct.field[].id 750623154 -documenttype[].datatype[].sstruct.field[].datatype 746267614 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].sstruct.field[].name "f2" -documenttype[].datatype[].sstruct.field[].id 1523850983 -documenttype[].datatype[].sstruct.field[].datatype 1811766610 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "f1" -documenttype[].fieldsets{[document]}.fields[] "f2" +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "simple" +doctype[].idx 10015 +doctype[].internalid 485659380 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "f1" +doctype[].fieldsets{[document]}.fields[] "f2" +doctype[].structtype[].idx 10018 +doctype[].structtype[].name "base" +doctype[].structtype[].field[].name "name" +doctype[].structtype[].field[].internalid 1160796772 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid -1396204461 +doctype[].structtype[].idx 10017 +doctype[].structtype[].name "child" +doctype[].structtype[].inherits[].type 10018 +doctype[].structtype[].field[].name "age" +doctype[].structtype[].field[].internalid 1862473705 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 746267614 +doctype[].structtype[].idx 10019 +doctype[].structtype[].name "grandchild" +doctype[].structtype[].inherits[].type 10017 +doctype[].structtype[].field[].name "toy" +doctype[].structtype[].field[].internalid 536645790 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid 1811766610 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "simple.header" +doctype[].structtype[].field[].name "f1" +doctype[].structtype[].field[].internalid 750623154 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "f2" +doctype[].structtype[].field[].internalid 1523850983 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].internalid -2142109237 diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 879b455a711..9ee6a82245f 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -1,53 +1,87 @@ enablecompression false usev8geopositions false -documenttype[].id -1290043429 -documenttype[].name "tensor" -documenttype[].version 0 -documenttype[].headerstruct 2125927172 -documenttype[].bodystruct 0 -documenttype[].inherits[].id 8 -documenttype[].datatype[].id 2125927172 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "tensor.header" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 -documenttype[].datatype[].sstruct.field[].name "f1" -documenttype[].datatype[].sstruct.field[].id 26661415 -documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[3])" -documenttype[].datatype[].sstruct.field[].name "f2" -documenttype[].datatype[].sstruct.field[].id 2080644671 -documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor<float>(x[2],y[1])" -documenttype[].datatype[].sstruct.field[].name "f3" -documenttype[].datatype[].sstruct.field[].id 1295091863 -documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x{})" -documenttype[].datatype[].sstruct.field[].name "f4" -documenttype[].datatype[].sstruct.field[].id 1224191509 -documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[10])" -documenttype[].datatype[].sstruct.field[].name "f5" -documenttype[].datatype[].sstruct.field[].id 329055840 -documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor<float>(x[10])" -documenttype[].datatype[].sstruct.field[].name "f6" -documenttype[].datatype[].sstruct.field[].id 596352344 -documenttype[].datatype[].sstruct.field[].datatype 1 -documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].fieldsets{[document]}.fields[] "f1" -documenttype[].fieldsets{[document]}.fields[] "f2" -documenttype[].fieldsets{[document]}.fields[] "f3" -documenttype[].fieldsets{[document]}.fields[] "f4" -documenttype[].fieldsets{[document]}.fields[] "f5" -documenttype[].fieldsets{[document]}.fields[] "f6" +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "tensor" +doctype[].idx 10015 +doctype[].internalid -1290043429 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "f1" +doctype[].fieldsets{[document]}.fields[] "f2" +doctype[].fieldsets{[document]}.fields[] "f3" +doctype[].fieldsets{[document]}.fields[] "f4" +doctype[].fieldsets{[document]}.fields[] "f5" +doctype[].fieldsets{[document]}.fields[] "f6" +doctype[].tensortype[].idx 10017 +doctype[].tensortype[].detailedtype "tensor(x[3])" +doctype[].tensortype[].idx 10018 +doctype[].tensortype[].detailedtype "tensor<float>(x[2],y[1])" +doctype[].tensortype[].idx 10019 +doctype[].tensortype[].detailedtype "tensor(x{})" +doctype[].tensortype[].idx 10020 +doctype[].tensortype[].detailedtype "tensor(x[10],y[10])" +doctype[].tensortype[].idx 10021 +doctype[].tensortype[].detailedtype "tensor<float>(x[10])" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "tensor.header" +doctype[].structtype[].field[].name "f1" +doctype[].structtype[].field[].internalid 26661415 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "f2" +doctype[].structtype[].field[].internalid 2080644671 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "f3" +doctype[].structtype[].field[].internalid 1295091863 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "f4" +doctype[].structtype[].field[].internalid 1224191509 +doctype[].structtype[].field[].type 10020 +doctype[].structtype[].field[].name "f5" +doctype[].structtype[].field[].internalid 329055840 +doctype[].structtype[].field[].type 10021 +doctype[].structtype[].field[].name "f6" +doctype[].structtype[].field[].internalid 596352344 +doctype[].structtype[].field[].type 10005 +doctype[].structtype[].internalid 2125927172 diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index 76d4ae3f1a6..aa70bf4d26a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -3,6 +3,10 @@ package com.yahoo.config.provision; import com.yahoo.cloud.config.ApplicationIdConfig; +import java.util.Comparator; +import java.util.Objects; +import java.util.regex.Pattern; + /** * A complete, immutable identification of an application instance. * @@ -10,27 +14,40 @@ import com.yahoo.cloud.config.ApplicationIdConfig; * @author vegard * @author bratseth */ -public final class ApplicationId implements Comparable<ApplicationId> { +public class ApplicationId implements Comparable<ApplicationId> { + + // TODO: remove '.' and '*' from this pattern. + static final Pattern namePattern = Pattern.compile("(?!\\.\\.)[a-zA-Z0-9_.*-]{1,256}"); + + private static final ApplicationId global = new ApplicationId(TenantName.from("*"), + ApplicationName.from("*"), + InstanceName.from("*")) { + @Override public boolean equals(Object other) { return this == other; } + }; + + private static final Comparator<ApplicationId> comparator = Comparator.comparing(ApplicationId::tenant) + .thenComparing(ApplicationId::application) + .thenComparing(ApplicationId::instance) + .thenComparing(global::equals, Boolean::compare); private final TenantName tenant; private final ApplicationName application; private final InstanceName instance; - - private final String stringValue; private final String serializedForm; - public ApplicationId(ApplicationIdConfig config) { - this(TenantName.from(config.tenant()), ApplicationName.from(config.application()), InstanceName.from(config.instance())); - } - private ApplicationId(TenantName tenant, ApplicationName applicationName, InstanceName instanceName) { this.tenant = tenant; this.application = applicationName; this.instance = instanceName; - this.stringValue = toStringValue(); this.serializedForm = toSerializedForm(); } + public static ApplicationId from(ApplicationIdConfig config) { + return from(TenantName.from(config.tenant()), + ApplicationName.from(config.application()), + InstanceName.from(config.instance())); + } + public static ApplicationId from(TenantName tenant, ApplicationName application, InstanceName instance) { return new ApplicationId(tenant, application, instance); } @@ -44,7 +61,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { if (parts.length < 3) throw new IllegalArgumentException("Application ids must be on the form tenant:application:instance, but was " + idString); - return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); + return from(parts[0], parts[1], parts[2]); } public static ApplicationId fromFullString(String idString) { @@ -52,11 +69,11 @@ public final class ApplicationId implements Comparable<ApplicationId> { if (parts.length < 3) throw new IllegalArgumentException("Application ids must be on the form tenant.application.instance, but was " + idString); - return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); + return from(parts[0], parts[1], parts[2]); } @Override - public int hashCode() { return stringValue.hashCode(); } + public int hashCode() { return Objects.hash(tenant, application, instance); } @Override public boolean equals(Object other) { @@ -72,10 +89,6 @@ public final class ApplicationId implements Comparable<ApplicationId> { /** Returns a serialized form of the content of this: tenant:application:instance */ public String serializedForm() { return serializedForm; } - private String toStringValue() { - return "tenant '" + tenant + "', application '" + application + "', instance '" + instance + "'"; - } - /** Returns "dotted" string (tenant.application.instance) with instance name omitted if it is "default" */ public String toShortString() { return tenant().value() + "." + application().value() + @@ -88,7 +101,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { } private String toSerializedForm() { - return tenant + ":" + application + ":" + instance; + return tenant.value() + ":" + application.value() + ":" + instance.value(); } @Override @@ -100,18 +113,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { @Override public int compareTo(ApplicationId other) { - int diff; - - diff = tenant.compareTo(other.tenant); - if (diff != 0) { return diff; } - - diff = application.compareTo(other.application); - if (diff != 0) { return diff; } - - diff = instance.compareTo(other.instance); - if (diff != 0) { return diff; } - - return 0; + return comparator.compare(this, other); } /** Returns an application id where all fields are "default" */ @@ -119,12 +121,10 @@ public final class ApplicationId implements Comparable<ApplicationId> { return new ApplicationId(TenantName.defaultName(), ApplicationName.defaultName(), InstanceName.defaultName()); } - /** Returns an application id where all fields are "*" */ + // TODO: kill this + /** Returns a very special application id, which is not equal to any other id. */ public static ApplicationId global() { - return new Builder().tenant("*") - .applicationName("*") - .instanceName("*") - .build(); + return global; } public static class Builder { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java index f16c126dec2..f2585913015 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationName.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; /** * Represents an applications name, which may be any kind of string or default. This type is defined @@ -10,28 +10,12 @@ import java.util.Objects; * @author Ulf Lilleengen * @since 5.25 */ -public class ApplicationName implements Comparable<ApplicationName> { +public class ApplicationName extends PatternedStringWrapper<ApplicationName> { - private final String applicationName; + private static final ApplicationName defaultName = new ApplicationName("default"); - private ApplicationName(String applicationName) { - this.applicationName = applicationName; - } - - @Override - public int hashCode() { - return applicationName.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ApplicationName)) return false; - return Objects.equals(((ApplicationName) obj).applicationName, applicationName); - } - - @Override - public String toString() { - return applicationName; + private ApplicationName(String name) { + super(name, ApplicationId.namePattern, "application name"); } public static ApplicationName from(String name) { @@ -39,20 +23,11 @@ public class ApplicationName implements Comparable<ApplicationName> { } public static ApplicationName defaultName() { - return new ApplicationName("default"); + return defaultName; } public boolean isDefault() { - return equals(ApplicationName.defaultName()); - } - - public String value() { - return applicationName; - } - - @Override - public int compareTo(ApplicationName name) { - return this.applicationName.compareTo(name.applicationName); + return equals(defaultName); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java index 8101b70b943..fc40d351465 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; /** * Represents an applications instance name, which may be any kind of string or default. This type is defined @@ -9,30 +9,12 @@ import java.util.Objects; * * @author Ulf Lilleengen */ -public class InstanceName implements Comparable<InstanceName> { +public class InstanceName extends PatternedStringWrapper<InstanceName> { - private static final InstanceName defaultInstance = new InstanceName("default"); + private static final InstanceName defaultName = new InstanceName("default"); - private final String instanceName; - - private InstanceName(String instanceName) { - this.instanceName = instanceName; - } - - @Override - public int hashCode() { - return instanceName.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof InstanceName)) return false; - return Objects.equals(((InstanceName) obj).instanceName, instanceName); - } - - @Override - public String toString() { - return instanceName; + private InstanceName(String name) { + super(name, ApplicationId.namePattern, "instance name"); } public static InstanceName from(String name) { @@ -40,22 +22,15 @@ public class InstanceName implements Comparable<InstanceName> { } public static InstanceName defaultName() { - return defaultInstance; + return defaultName; } public boolean isDefault() { - return equals(InstanceName.defaultName()); + return equals(defaultName); } public boolean isTester() { return value().endsWith("-t"); } - public String value() { return instanceName; } - - @Override - public int compareTo(InstanceName instance) { - return instanceName.compareTo(instance.instanceName); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java index 92fe5345b4e..9909ab360a0 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java @@ -1,56 +1,31 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import java.util.Objects; +import ai.vespa.validation.PatternedStringWrapper; /** * Represents a tenant in the provision API. * * @author Ulf Lilleengen */ -public class TenantName implements Comparable<TenantName> { +public class TenantName extends PatternedStringWrapper<TenantName> { - private final String name; + private static final TenantName defaultName = new TenantName("default"); private TenantName(String name) { - this.name = name; + super(name, ApplicationId.namePattern, "tenant name"); } - public String value() { return name; } - - /** - * Create a {@link TenantName} with a given name. - * - * @param name Name of tenant. - * @return instance of {@link TenantName}. - */ public static TenantName from(String name) { return new TenantName(name); } - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof TenantName)) return false; - return Objects.equals(((TenantName)obj).value(), value()); - } - - @Override - public String toString() { - return name; - } - public static TenantName defaultName() { - return from("default"); + return defaultName; } - @Override - public int compareTo(TenantName that) { - return this.name.compareTo(that.name); + public boolean isDefault() { + return equals(defaultName); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java index c82230f7edf..01904b5eece 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java @@ -106,7 +106,7 @@ public class ApplicationIdTest { builder.tenant("a"); builder.application("b"); builder.instance("c"); - ApplicationId applicationId = new ApplicationId(new ApplicationIdConfig(builder)); + ApplicationId applicationId = ApplicationId.from(new ApplicationIdConfig(builder)); assertEquals("a", applicationId.tenant().value()); assertEquals("b", applicationId.application().value()); assertEquals("c", applicationId.instance().value()); diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 05143bfef9f..64ca1e522f5 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -20,6 +20,10 @@ configServerDBDir string default="var/db/vespa/config_server/serverdb/" configDefinitionsDir string default="share/vespa/configdefinitions/" fileReferencesDir string default="var/db/vespa/filedistribution/" +# Application package +# The maximum decompressed size of an application package, in bytes. Defaults to 8 GB +maxApplicationPackageSize long default=8589934592 + # Misc sessionLifetime long default=3600 # in seconds masterGeneration long default=0 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 7a754dd84cd..76d7ff2fc7f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -31,6 +31,7 @@ import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.path.Path; +import com.yahoo.restapi.HttpURL; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -558,27 +559,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public HttpResponse serviceStatusPage(ApplicationId applicationId, String hostName, String serviceName, String pathSuffix) { + public HttpResponse serviceStatusPage(ApplicationId applicationId, String hostName, String serviceName, HttpURL.Path pathSuffix) { // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise // change the hostname and port. Exposing other paths on the cluster controller should be fine. // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. - String pathPrefix; + HttpURL.Path pathPrefix = HttpURL.Path.empty(); switch (serviceName) { - case "container-clustercontroller": { - pathPrefix = "clustercontroller-status/v1/"; + case "container-clustercontroller": + pathPrefix = pathPrefix.append("clustercontroller-status").append("v1"); break; - } case "distributor": - case "storagenode": { - pathPrefix = ""; + case "storagenode": break; - } default: throw new NotFoundException("No status page for service: " + serviceName); } - return httpProxy.get(getApplication(applicationId), hostName, serviceName, pathPrefix + pathSuffix); + return httpProxy.get(getApplication(applicationId), hostName, serviceName, pathPrefix.append(pathSuffix)); } public Map<String, ClusterReindexing> getClusterReindexingStatus(ApplicationId applicationId) { @@ -659,9 +657,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getOptionalApplication(applicationId).map(app -> app.getModel().fileReferences()).orElse(Set.of()); } - public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, Session.Mode mode) { + public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, HttpURL.Path path, Session.Mode mode) { Tenant tenant = tenantRepository.getTenant(tenantName); - return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); + return getLocalSession(tenant, sessionId).getApplicationFile(Path.from(path.segments()), mode); } public Tenant getTenant(ApplicationId applicationId) { @@ -1053,7 +1051,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = - CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { + CompressedApplicationInputStream.createFromCompressedStream(in, contentType, configserverConfig.maxApplicationPackageSize())) { return decompressApplication(application, tempDir); } catch (IOException e) { throw new IllegalArgumentException("Unable to decompress data in body", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 0672f13fd6a..443ab47e786 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -1,23 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.google.common.io.ByteStreams; +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.logging.Level; -import com.yahoo.vespa.config.server.http.BadRequestException; -import com.yahoo.vespa.config.server.http.InternalServerException; -import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; - import java.util.logging.Logger; -import java.util.zip.GZIPInputStream; import static com.yahoo.yolean.Exceptions.uncheck; @@ -29,53 +27,43 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class CompressedApplicationInputStream implements AutoCloseable { private static final Logger log = Logger.getLogger(CompressedApplicationInputStream.class.getPackage().getName()); - private final ArchiveInputStream ais; + + private final ArchiveStreamReader reader; + + private CompressedApplicationInputStream(ArchiveStreamReader reader) { + this.reader = reader; + } /** * Create an instance of a compressed application from an input stream. * * @param is the input stream containing the compressed files. * @param contentType the content type for determining what kind of compressed stream should be used. + * @param maxSizeInBytes the maximum allowed size of the decompressed content * @return An instance of an unpacked application. */ - public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType) { + public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType, long maxSizeInBytes) { try { - ArchiveInputStream ais = getArchiveInputStream(is, contentType); - return createFromCompressedStream(ais); - } catch (IOException e) { + Options options = Options.standard().maxSize(maxSizeInBytes).allowDotSegment(true); + switch (contentType) { + case ApplicationApiHandler.APPLICATION_X_GZIP: + return new CompressedApplicationInputStream(ArchiveStreamReader.ofTarGzip(is, options)); + case ApplicationApiHandler.APPLICATION_ZIP: + return new CompressedApplicationInputStream(ArchiveStreamReader.ofZip(is, options)); + default: + throw new BadRequestException("Unable to decompress"); + } + } catch (UncheckedIOException e) { throw new InternalServerException("Unable to create compressed application stream", e); } } - static CompressedApplicationInputStream createFromCompressedStream(ArchiveInputStream ais) { - return new CompressedApplicationInputStream(ais); - } - - private static ArchiveInputStream getArchiveInputStream(InputStream is, String contentTypeHeader) throws IOException { - ArchiveInputStream ais; - switch (contentTypeHeader) { - case ApplicationApiHandler.APPLICATION_X_GZIP: - ais = new TarArchiveInputStream(new GZIPInputStream(is)); - break; - case ApplicationApiHandler.APPLICATION_ZIP: - ais = new ZipArchiveInputStream(is); - break; - default: - throw new BadRequestException("Unable to decompress"); - } - return ais; - } - - private CompressedApplicationInputStream(ArchiveInputStream ais) { - this.ais = ais; - } - /** * Close this stream. * @throws IOException if the stream could not be closed */ public void close() throws IOException { - ais.close(); + reader.close(); } File decompress() throws IOException { @@ -83,45 +71,44 @@ public class CompressedApplicationInputStream implements AutoCloseable { } public File decompress(File dir) throws IOException { - decompressInto(dir); + decompressInto(dir.toPath()); dir = findActualApplicationDir(dir); return dir; } - private void decompressInto(File application) throws IOException { - log.log(Level.FINE, () -> "Application is in " + application.getAbsolutePath()); + private void decompressInto(Path dir) throws IOException { + if (!Files.isDirectory(dir)) throw new IllegalArgumentException("Not a directory: " + dir.toAbsolutePath()); + log.log(Level.FINE, () -> "Application is in " + dir.toAbsolutePath()); int entries = 0; - ArchiveEntry entry; - while ((entry = ais.getNextEntry()) != null) { - log.log(Level.FINE, "Unpacking %s", entry.getName()); - File outFile = new File(application, entry.getName()); - // FIXME/TODO: write more tests that break this logic. I have a feeling it is not very robust. - if (entry.isDirectory()) { - if (!(outFile.exists() && outFile.isDirectory())) { - log.log(Level.FINE, () -> "Creating dir: " + outFile.getAbsolutePath()); - boolean res = outFile.mkdirs(); - if (!res) { - log.log(Level.WARNING, "Could not create dir " + entry.getName()); - } - } - } else { - log.log(Level.FINE, () -> "Creating output file: " + outFile.getAbsolutePath()); - - // Create parent dir if necessary - String parent = outFile.getParent(); - new File(parent).mkdirs(); - - FileOutputStream fos = new FileOutputStream(outFile); - ByteStreams.copy(ais, fos); - fos.close(); + Path tmpFile = null; + OutputStream tmpStream = null; + try { + tmpFile = createTempFile(dir); + tmpStream = Files.newOutputStream(tmpFile); + ArchiveStreamReader.ArchiveFile file; + while ((file = reader.readNextTo(tmpStream)) != null) { + tmpStream.close(); + log.log(Level.FINE, "Creating output file: " + file.path()); + Path dstFile = dir.resolve(file.path().toString()).normalize(); + Files.createDirectories(dstFile.getParent()); + Files.move(tmpFile, dstFile); + tmpFile = createTempFile(dir); + tmpStream = Files.newOutputStream(tmpFile); + entries++; } - entries++; + } finally { + if (tmpStream != null) tmpStream.close(); + if (tmpFile != null) Files.deleteIfExists(tmpFile); } if (entries == 0) { - log.log(Level.WARNING, "Not able to read any entries from " + application.getName()); + log.log(Level.WARNING, "Not able to decompress any entries to " + dir); } } + private static Path createTempFile(Path applicationDir) throws IOException { + return Files.createTempFile(applicationDir, "application", null); + } + private File findActualApplicationDir(File application) { // If application is in e.g. application/, use that as root for UnpackedApplication // TODO: Vespa 8: Remove application/ directory support @@ -131,4 +118,5 @@ public class CompressedApplicationInputStream implements AutoCloseable { } return application; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index a8915d187f3..14acd1b5630 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -10,8 +10,14 @@ import com.yahoo.container.jdisc.HttpResponse; import java.util.HashSet; import java.util.List; import java.util.logging.Level; + +import com.yahoo.net.DomainName; +import com.yahoo.restapi.HttpURL; +import com.yahoo.restapi.HttpURL.Path; +import com.yahoo.restapi.HttpURL.Scheme; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; +import com.yahoo.vespa.config.server.http.HttpFetcher.Params; import com.yahoo.vespa.config.server.http.NotFoundException; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; @@ -33,7 +39,7 @@ public class HttpProxy { this.fetcher = fetcher; } - public HttpResponse get(Application application, String hostName, String serviceType, String relativePath) { + public HttpResponse get(Application application, String hostName, String serviceType, Path relativePath) { HostInfo host = application.getModel().getHosts().stream() .filter(hostInfo -> hostInfo.getHostname().equals(hostName)) .findFirst() @@ -54,18 +60,15 @@ public class HttpProxy { return internalGet(host.getHostname(), port.getPort(), relativePath); } - private HttpResponse internalGet(String hostname, int port, String relativePath) { - String urlString = "http://" + hostname + ":" + port + "/" + relativePath; - URL url; + private HttpResponse internalGet(String hostname, int port, Path relativePath) { + HttpURL url = HttpURL.create(Scheme.http, DomainName.of(hostname), port, relativePath); try { - url = new URL(urlString); + return fetcher.get(new Params(2000), // 2_000 ms read timeout + url.asURI().toURL()); } catch (MalformedURLException e) { - logger.log(Level.WARNING, "Badly formed url: " + urlString, e); + logger.log(Level.WARNING, "Badly formed url: " + url, e); return HttpErrorResponse.internalServerError("Failed to construct URL for backend"); } - - HttpFetcher.Params params = new HttpFetcher.Params(2000); // 2_000 ms read timeout - return fetcher.get(params, url); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 0ee9bc84ff7..fe9ab82637f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -211,7 +211,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useQrserverServiceName; private final boolean avoidRenamingSummaryFeatures; private final boolean experimentalSdParsing; - private final Architecture adminClusterNodeResourcesArchitecture; + private final Architecture adminClusterArchitecture; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -260,7 +260,7 @@ public class ModelContextImpl implements ModelContext { this.useQrserverServiceName = flagValue(source, appId, version, Flags.USE_QRSERVER_SERVICE_NAME); this.avoidRenamingSummaryFeatures = flagValue(source, appId, version, Flags.AVOID_RENAMING_SUMMARY_FEATURES); this.experimentalSdParsing = flagValue(source, appId, version, Flags.EXPERIMENTAL_SD_PARSING); - this.adminClusterNodeResourcesArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); + this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -311,8 +311,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } - @Override public String adminClusterNodeArchitecture() { return adminClusterArchitecture().name(); } - @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } + @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java index 13acc121fa0..0ceb459233b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; @@ -32,7 +33,7 @@ public class ContentHandler { public HttpResponse put(ContentRequest request) { ApplicationFile file = request.getFile(); - if (request.getPath().endsWith("/")) { + if (request.getPath().hasTrailingSlash()) { createDirectory(request, file); } else { createFile(request, file); @@ -62,9 +63,9 @@ public class ContentHandler { return new SessionContentStatusResponse(file, urlBase); } - private static List<ApplicationFile> listSortedFiles(ApplicationFile file, String path, boolean recursive) { - if (!path.isEmpty() && !path.endsWith("/")) { - return Arrays.asList(file); + private static List<ApplicationFile> listSortedFiles(ApplicationFile file, Path path, boolean recursive) { + if ( ! path.segments().isEmpty() && ! path.hasTrailingSlash()) { + return List.of(file); } List<ApplicationFile> files = file.listFiles(recursive); Collections.sort(files); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java index 995c4b2dc56..f06e1dabf8c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.HttpURL.Path; import java.io.InputStream; @@ -20,11 +21,11 @@ public abstract class ContentRequest { enum ReturnType {CONTENT, STATUS} private final long sessionId; - private final String path; + private final Path path; private final ApplicationFile file; private final HttpRequest request; - protected ContentRequest(HttpRequest request, long sessionId, String path, ApplicationFile applicationFile) { + protected ContentRequest(HttpRequest request, long sessionId, Path path, ApplicationFile applicationFile) { this.request = request; this.sessionId = sessionId; this.path = path; @@ -77,7 +78,7 @@ public abstract class ContentRequest { } - String getPath() { + Path getPath() { return path; } @@ -86,8 +87,8 @@ public abstract class ContentRequest { } ApplicationFile getExistingFile() { - if (!file.exists()) { - throw new NotFoundException("Session " + sessionId + " does not contain a file '" + path + "'"); + if ( ! file.exists()) { + throw new NotFoundException("Session " + sessionId + " does not contain a file at " + path); } return file; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index 1a8b36ee19f..c8953d5996c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -49,9 +49,11 @@ public class ApplicationApiHandler extends SessionHandler { public final static String MULTIPART_PARAMS = "prepareParams"; public final static String MULTIPART_APPLICATION_PACKAGE = "applicationPackage"; public final static String contentTypeHeader = "Content-Type"; + private final TenantRepository tenantRepository; private final Duration zookeeperBarrierTimeout; private final Zone zone; + private final long maxApplicationPackageSize; @Inject public ApplicationApiHandler(Context ctx, @@ -61,6 +63,7 @@ public class ApplicationApiHandler extends SessionHandler { super(ctx, applicationRepository); this.tenantRepository = applicationRepository.tenantRepository(); this.zookeeperBarrierTimeout = Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()); + this.maxApplicationPackageSize = configserverConfig.maxApplicationPackageSize(); this.zone = zone; } @@ -85,14 +88,14 @@ public class ApplicationApiHandler extends SessionHandler { log.log(Level.FINE, "Deploy parameters: [{0}]", new String(params, StandardCharsets.UTF_8)); prepareParams = PrepareParams.fromJson(params, tenantName, zookeeperBarrierTimeout); Part appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); - compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType()); + compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType(), maxApplicationPackageSize); } catch (IOException e) { log.log(Level.WARNING, "Unable to parse multipart in deploy", e); throw new BadRequestException("Request contains invalid data"); } } else { prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); + compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader), maxApplicationPackageSize); } PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 0707261bb45..885456ff69c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -16,6 +16,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.HttpURL; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; @@ -132,23 +133,23 @@ public class ApplicationHandler extends HttpHandler { return HttpServiceResponse.createResponse(response, hostAndPort, request.getUri()); } - private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, String pathSuffix) { + private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, HttpURL.Path pathSuffix) { return applicationRepository.serviceStatusPage(applicationId, hostname, service, pathSuffix); } - private HttpResponse content(ApplicationId applicationId, String contentPath, HttpRequest request) { + private HttpResponse content(ApplicationId applicationId, HttpURL.Path contentPath, HttpRequest request) { long sessionId = applicationRepository.getSessionIdForApplication(applicationId); ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(applicationId.tenant(), - sessionId, - contentPath, - ContentRequest.getApplicationFileMode(request.getMethod())); + sessionId, + contentPath, + ContentRequest.getApplicationFileMode(request.getMethod())); ApplicationContentRequest contentRequest = new ApplicationContentRequest(request, - sessionId, - applicationId, - zone, - contentPath, - applicationFile); + sessionId, + applicationId, + zone, + contentPath, + applicationFile); return new ContentHandler().get(contentRequest); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java index 17a8e1449c4..f6af9e616a9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.ContentHandler; @@ -52,7 +53,7 @@ public class SessionContentHandler extends SessionHandler { TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); validateRequest(tenantName); long sessionId = getSessionIdV2(request); - String contentPath = SessionContentRequestV2.getContentPath(request); + Path contentPath = SessionContentRequestV2.getContentPath(request); ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(tenantName, sessionId, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java index 34c842eca44..15d6c5c18ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.vespa.config.server.http.ContentRequest; /** @@ -22,7 +23,7 @@ public class ApplicationContentRequest extends ContentRequest { long sessionId, ApplicationId applicationId, Zone zone, - String contentPath, + Path contentPath, ApplicationFile applicationFile) { super(request, sessionId, contentPath, applicationFile); this.applicationId = applicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java index da71fd89054..449058eb911 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java @@ -5,6 +5,8 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.restapi.HttpURL; +import com.yahoo.restapi.Path; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.Utils; @@ -16,14 +18,14 @@ import com.yahoo.vespa.config.server.http.Utils; * @since 5.3 */ public class SessionContentRequestV2 extends ContentRequest { - private static final String uriPattern = "http://*/application/v2/tenant/*/session/*/content/*"; + private final TenantName tenantName; private final long sessionId; public SessionContentRequestV2(HttpRequest request, long sessionId, TenantName tenantName, - String path, + HttpURL.Path path, ApplicationFile applicationFile) { super(request, sessionId, path, applicationFile); this.tenantName = tenantName; @@ -35,8 +37,11 @@ public class SessionContentRequestV2 extends ContentRequest { return "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; } - public static String getContentPath(HttpRequest request) { - BindingMatch<?> bm = Utils.getBindingMatch(request, uriPattern); - return bm.group(4); + public static HttpURL.Path getContentPath(HttpRequest request) { + Path path = new Path(request.getUri()); + if ( ! path.matches("/application/v2/tenant/{tenant}/session/{session}/content/{*}")) + throw new IllegalStateException("error in request routing"); + return path.getRest(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index d3976464bde..7e6fccb6d2f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -193,10 +193,9 @@ public class DelayedConfigResponses { } private synchronized void metricDelayedResponses(ApplicationId app, int elems) { - if ( ! metrics.containsKey(app)) { - metrics.put(app, rpcServer.metricUpdaterFactory().getOrCreateMetricUpdater(Metrics.createDimensions(app))); - } - metrics.get(app).setDelayedResponses(elems); + metrics.computeIfAbsent(app, key -> rpcServer.metricUpdaterFactory() + .getOrCreateMetricUpdater(Metrics.createDimensions(key))) + .setDelayedResponses(elems); } private synchronized void createQueueIfNotExists(GetConfigContext context) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 99ffff6403b..ebf1fb32141 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -243,13 +243,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } private ApplicationState getState(ApplicationId id) { - ApplicationState state = applicationStateMap.get(id); - if (state == null) { - applicationStateMap.putIfAbsent(id, new ApplicationState(0)); - state = applicationStateMap.get(id); - } - return state; + return applicationStateMap.computeIfAbsent(id, __ -> new ApplicationState(0)); } + boolean hasNewerGeneration(ApplicationId id, long generation) { return getState(id).getActiveGeneration() > generation; } diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java index 30975be61e2..8b9714c3bfb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java @@ -5,6 +5,11 @@ import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.net.DomainName; +import com.yahoo.restapi.HttpURL; +import com.yahoo.restapi.HttpURL.Path; +import com.yahoo.restapi.HttpURL.Query; +import com.yahoo.restapi.HttpURL.Scheme; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.restapi.UriBuilder; @@ -106,7 +111,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl String regionName = context.pathParameters().getStringOrThrow("regionName"); String instanceName = context.pathParameters().getStringOrThrow("instanceName"); String identifier = context.pathParameters().getStringOrThrow("serviceIdentifier"); - String apiParams = context.pathParameters().getString("*").orElse(""); + Path apiParams = context.pathParameters().getRest().orElse(Path.empty()); return singleService(context.uriBuilder(), context.request().getUri(), tenantName, applicationName, environmentName, regionName, instanceName, identifier, apiParams); } @@ -125,7 +130,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl } protected HashMap<?, ?> singleService( - UriBuilder uriBuilder, URI requestUri, String tenantName, String applicationName, String environmentName, String regionName, String instanceName, String identifier, String apiParams) { + UriBuilder uriBuilder, URI requestUri, String tenantName, String applicationName, String environmentName, String regionName, String instanceName, String identifier, Path apiParams) { ServiceModel model = new ServiceModel(getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); Service s = model.getService(identifier); int requestedPort = s.matchIdentifierWithPort(identifier); @@ -135,11 +140,9 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl return apiResult; } - protected HealthClient getHealthClient(String apiParams, Service s, int requestedPort, String uriQuery, Client client) { - final StringBuilder uriBuffer = new StringBuilder("http://").append(s.host).append(':').append(requestedPort).append('/') - .append(apiParams); - addQuery(uriQuery, uriBuffer); - WebTarget target = client.target(uriBuffer.toString()); + protected HealthClient getHealthClient(Path apiParams, Service s, int requestedPort, String uriQuery, Client client) { + URI uri = HttpURL.create(Scheme.http, DomainName.of(s.host), requestedPort, apiParams, Query.parse(uriQuery)).asURI(); + WebTarget target = client.target(uri); return WebResourceFactory.newResource(HealthClient.class, target); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 23444ac53d6..c7662ac9ee4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -2,24 +2,27 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.yolean.Exceptions; import org.apache.commons.compress.archivers.ArchiveOutputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -27,6 +30,9 @@ import static org.junit.Assert.assertTrue; */ public class CompressedApplicationInputStreamTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private static void writeFileToTar(ArchiveOutputStream taos, File file) throws IOException { taos.putArchiveEntry(taos.createArchiveEntry(file, file.getName())); ByteStreams.copy(new FileInputStream(file), taos); @@ -42,14 +48,14 @@ public class CompressedApplicationInputStreamTest { return outFile; } - public static File createTarFile() throws IOException { - File outFile = File.createTempFile("testapp", ".tar.gz"); + public static File createTarFile(Path dir) throws IOException { + File outFile = Files.createTempFile(dir, "testapp", ".tar.gz").toFile(); ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); return createArchiveFile(archiveOutputStream, outFile); } - private static File createZipFile() throws IOException { - File outFile = File.createTempFile("testapp", ".tar.gz"); + private File createZipFile(Path dir) throws IOException { + File outFile = Files.createTempFile(dir, "testapp", ".tar.gz").toFile(); ArchiveOutputStream archiveOutputStream = new ZipArchiveOutputStream(new FileOutputStream(outFile)); return createArchiveFile(archiveOutputStream, outFile); } @@ -63,15 +69,16 @@ public class CompressedApplicationInputStreamTest { @Test public void require_that_valid_tar_application_can_be_unpacked() throws IOException { - File outFile = createTarFile(); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(outFile)))); - File outApp = unpacked.decompress(); - assertTestApp(outApp); + File outFile = createTarFile(temporaryFolder.getRoot().toPath()); + try (CompressedApplicationInputStream unpacked = streamFromTarGz(outFile)) { + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } } @Test public void require_that_valid_tar_application_in_subdir_can_be_unpacked() throws IOException { - File outFile = File.createTempFile("testapp", ".tar.gz"); + File outFile = Files.createTempFile(temporaryFolder.getRoot().toPath(), "testapp", ".tar.gz").toFile(); ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); File app = new File("src/test/resources/deploy/validapp"); @@ -91,48 +98,39 @@ public class CompressedApplicationInputStreamTest { archiveOutputStream.close(); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(outFile)))); - File outApp = unpacked.decompress(); - assertEquals("application", outApp.getName()); // gets the name of the subdir - assertTestApp(outApp); + try (CompressedApplicationInputStream unpacked = streamFromTarGz(outFile)) { + File outApp = unpacked.decompress(); + assertEquals("application", outApp.getName()); // gets the name of the subdir + assertTestApp(outApp); + } } @Test public void require_that_valid_zip_application_can_be_unpacked() throws IOException { - File outFile = createZipFile(); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( - new ZipArchiveInputStream(new FileInputStream(outFile))); - File outApp = unpacked.decompress(); - assertTestApp(outApp); + File outFile = createZipFile(temporaryFolder.getRoot().toPath()); + try (CompressedApplicationInputStream unpacked = streamFromZip(outFile)) { + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } } @Test public void require_that_gnu_tared_file_can_be_unpacked() throws IOException, InterruptedException { - File tmpTar = File.createTempFile("myapp", ".tar"); - Process p = new ProcessBuilder("tar", "-C", "src/test/resources/deploy/validapp", "--exclude=.svn", "-cvf", tmpTar.getAbsolutePath(), ".").start(); - p.waitFor(); - p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); - p.waitFor(); - File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + File gzFile = createTarGz("src/test/resources/deploy/validapp"); assertTrue(gzFile.exists()); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( - new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(gzFile), "application/x-gzip", Long.MAX_VALUE); File outApp = unpacked.decompress(); assertTestApp(outApp); } @Test public void require_that_nested_app_can_be_unpacked() throws IOException, InterruptedException { - File tmpTar = File.createTempFile("myapp", ".tar"); - Process p = new ProcessBuilder("tar", "-C", "src/test/resources/deploy/advancedapp", "--exclude=.svn", "-cvf", tmpTar.getAbsolutePath(), ".").start(); - p.waitFor(); - p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); - p.waitFor(); - File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + File gzFile = createTarGz("src/test/resources/deploy/advancedapp"); assertTrue(gzFile.exists()); - CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( - new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); - File outApp = unpacked.decompress(); + File outApp; + try (CompressedApplicationInputStream unpacked = streamFromTarGz(gzFile)) { + outApp = unpacked.decompress(); + } List<File> files = Arrays.asList(outApp.listFiles()); assertEquals(5, files.size()); assertTrue(files.contains(new File(outApp, "services.xml"))); @@ -164,11 +162,29 @@ public class CompressedApplicationInputStreamTest { assertEquals(new File(bar, "lol").getAbsolutePath(), bar.listFiles()[0].getAbsolutePath()); } - - @Test(expected = IOException.class) - public void require_that_invalid_application_returns_error_when_unpacked() throws IOException { + @Test(expected = InternalServerException.class) + public void require_that_invalid_application_returns_error_when_unpacked() throws Exception { File app = new File("src/test/resources/deploy/validapp/services.xml"); - CompressedApplicationInputStream.createFromCompressedStream( - new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(app)))); + streamFromTarGz(app).close(); } + + private File createTarGz(String appDir) throws IOException, InterruptedException { + File tmpTar = Files.createTempFile(temporaryFolder.getRoot().toPath(), "myapp", ".tar").toFile(); + Process p = new ProcessBuilder("tar", "-C", appDir, "-cvf", tmpTar.getAbsolutePath(), ".").start(); + p.waitFor(); + p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); + p.waitFor(); + File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + assertTrue(gzFile.exists()); + return gzFile; + } + + private static CompressedApplicationInputStream streamFromZip(File zipFile) { + return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(zipFile), "application/zip", Long.MAX_VALUE)); + } + + private static CompressedApplicationInputStream streamFromTarGz(File tarFile) { + return Exceptions.uncheck(() -> CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(tarFile), "application/x-gzip", Long.MAX_VALUE)); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index ac0a6491100..83cae04cbfd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.vespa.config.server.http.HttpFetcher; import com.yahoo.vespa.config.server.http.RequestTimeoutException; import com.yahoo.vespa.config.server.http.StaticResponse; @@ -48,7 +49,7 @@ public class HttpProxyTest { when(fetcher.get(actualParams.capture(), actualUrl.capture())).thenReturn(response); HttpResponse actualResponse = proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, - "clustercontroller-status/v1/clusterName"); + Path.parse("clustercontroller-status/v1/clusterName")); assertEquals(1, actualParams.getAllValues().size()); assertEquals(2000, actualParams.getValue().readTimeoutMs); @@ -67,7 +68,7 @@ public class HttpProxyTest { when(fetcher.get(any(), any())).thenThrow(new RequestTimeoutException("timed out")); proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, - "clustercontroller-status/v1/clusterName"); + Path.parse("clustercontroller-status/v1/clusterName")); } private static MockModel createClusterController() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java index 69caf86729f..0732379e0d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java @@ -34,7 +34,7 @@ public class HandlerTest { if (contentType != null) { assertEquals(renderedString, contentType, response.getContentType()); } - assertTrue(renderedString.contains(message)); + assertTrue("\n" + renderedString + "\n should contain \n" + message, renderedString.contains(message)); } public static void assertHttpStatusCodeErrorCodeAndMessage(HttpResponse response, int statusCode, HttpErrorResponse.ErrorCode errorCode, String message) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 005dd715dd4..9f7e539a2e3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -15,6 +15,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.restapi.HttpURL; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockLogRetriever; @@ -356,16 +357,20 @@ public class ApplicationHandlerTest { .withHttpProxy(mockHttpProxy) .build(); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); - doAnswer(invoc -> new StaticResponse(200, "text/html", "<html>" + - "host=" + invoc.getArgument(1, String.class) + "," + - "service=" + invoc.getArgument(2, String.class) + "," + - "path=" + invoc.getArgument(3, String.class) + "</html>")).when(mockHttpProxy).get(any(), any(), any(), any()); + doAnswer(invoc -> new StaticResponse(200, + "text/html", + "<html>" + + "host=" + invoc.getArgument(1, String.class) + "," + + "service=" + invoc.getArgument(2, String.class) + "," + + "path=" + invoc.getArgument(3, HttpURL.Path.class) + + "</html>")) + .when(mockHttpProxy).get(any(), any(), any(), any()); HttpResponse response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/container-clustercontroller/" + host + "/status/some/path/clusterName1", GET)); - assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=container-clustercontroller,path=clustercontroller-status/v1/some/path/clusterName1</html>"); + assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=container-clustercontroller,path=path '/clustercontroller-status/v1/some/path/clusterName1'</html>"); response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/distributor/" + host + "/status/something", GET)); - assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=something</html>"); + assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=path '/something'</html>"); response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/fake-service/" + host + "/status/something", GET)); assertHttpStatusCodeAndMessage(response, 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No status page for service: fake-service\"}"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 790be6d45ba..7c2e0be0c3a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -121,13 +121,13 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { @Test public void require_that_nonexistent_file_returns_not_found_when_deleted() throws IOException { - assertDeleteFile(Response.Status.NOT_FOUND, "/test2.txt", "{\"error-code\":\"NOT_FOUND\",\"message\":\"Session " + sessionId + " does not contain a file 'test2.txt'\"}"); + assertDeleteFile(Response.Status.NOT_FOUND, "/test2.txt", "{\"error-code\":\"NOT_FOUND\",\"message\":\"Session " + sessionId + " does not contain a file at path '/test2.txt'\"}"); } @Test public void require_that_files_can_be_deleted() throws IOException { assertDeleteFile(Response.Status.OK, "/test.txt"); - assertDeleteFile(Response.Status.NOT_FOUND, "/test.txt", "{\"error-code\":\"NOT_FOUND\",\"message\":\"Session " + sessionId + " does not contain a file 'test.txt'\"}"); + assertDeleteFile(Response.Status.NOT_FOUND, "/test.txt", "{\"error-code\":\"NOT_FOUND\",\"message\":\"Session " + sessionId + " does not contain a file at path '/test.txt'\"}"); assertDeleteFile(Response.Status.BAD_REQUEST, "/newtest", "{\"error-code\":\"BAD_REQUEST\",\"message\":\"File 'newtest' is not an empty directory\"}"); assertDeleteFile(Response.Status.OK, "/newtest/testfile.txt"); assertDeleteFile(Response.Status.OK, "/newtest"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 702dd2792da..58a3593ae14 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; @@ -109,20 +110,20 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Test public void require_that_post_request_must_have_correct_content_type() throws IOException { HashMap<String, String> headers = new HashMap<>(); // no Content-Type header - File outFile = CompressedApplicationInputStreamTest.createTarFile(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); HttpResponse response = createHandler().handle(post(outFile, headers, null)); assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.ErrorCode.BAD_REQUEST, "Request contains no Content-Type header"); } private void assertIllegalFromParameter(String fromValue) throws IOException { - File outFile = CompressedApplicationInputStreamTest.createTarFile(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); HttpRequest request = post(outFile, postHeaders, Collections.singletonMap("from", fromValue)); assertHttpStatusCodeErrorCodeAndMessage(createHandler().handle(request), BAD_REQUEST, HttpErrorResponse.ErrorCode.BAD_REQUEST, "Parameter 'from' has illegal value '" + fromValue + "'"); } @Test public void require_that_prepare_url_is_returned_on_success() throws IOException { - File outFile = CompressedApplicationInputStreamTest.createTarFile(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); Map<String, String> parameters = Collections.singletonMap("name", "foo"); HttpResponse response = createHandler().handle(post(outFile, postHeaders, parameters)); assertNotNull(response); @@ -143,7 +144,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Test public void require_internal_error_when_exception() throws IOException { - File outFile = CompressedApplicationInputStreamTest.createTarFile(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); new FileWriter(outFile).write("rubbish"); HttpResponse response = createHandler().handle(post(outFile)); assertHttpStatusCodeErrorCodeAndMessage(response, INTERNAL_SERVER_ERROR, @@ -153,9 +154,9 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Test public void require_that_handler_unpacks_application() throws IOException { - File outFile = CompressedApplicationInputStreamTest.createTarFile(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); createHandler().handle(post(outFile)); - ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(tenant, 2, "services.xml", Session.Mode.READ); + ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(tenant, 2, Path.parse("services.xml"), Session.Mode.READ); assertTrue(applicationFile.exists()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java index fe025c9e861..35e63c46bf8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.serviceview; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.restapi.UriBuilder; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.HealthClient; @@ -45,7 +46,7 @@ public class StateRequestHandlerTest { } @Override - protected HealthClient getHealthClient(String apiParams, Service s, int requestedPort, String uriQuery, Client client) { + protected HealthClient getHealthClient(Path apiParams, Service s, int requestedPort, String uriQuery, Client client) { HealthClient healthClient = Mockito.mock(HealthClient.class); HashMap<Object, Object> dummyHealthData = new HashMap<>(); HashMap<String, String> dummyLink = new HashMap<>(); @@ -75,7 +76,7 @@ public class StateRequestHandlerTest { public final void test() { Service s = correspondingModel.resolve("vespa.yahoo.com", 8080, null); String api = "/state/v1"; - HashMap<?, ?> boom = testHandler.singleService(new UriBuilder("http://someserver:8080"), URI.create(EXTERNAL_BASE_URI), "default", "default", "default", "default", "default", s.getIdentifier(8080), api); + HashMap<?, ?> boom = testHandler.singleService(new UriBuilder("http://someserver:8080"), URI.create(EXTERNAL_BASE_URI), "default", "default", "default", "default", "default", s.getIdentifier(8080), Path.parse(api)); assertEquals(EXTERNAL_BASE_URI + "tenant/default/application/default/environment/default/region/default/instance/default/service/" + s.getIdentifier(8080) + api, ((Map<?, ?>) ((List<?>) boom.get("resources")).get(0)).get("url")); } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index 44a70ea2f3b..be8ba669ec0 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -19,10 +19,10 @@ public class DocumentAccessProvider implements Provider<VespaDocumentAccess> { private final VespaDocumentAccess access; @Inject - public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, + public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { - this.access = new VespaDocumentAccess(documentmanagerConfig, loadTypeConfig, System.getProperty("config.id"), + this.access = new VespaDocumentAccess(documentmanagerConfig, System.getProperty("config.id"), messagebusConfig, policiesConfig, distributionConfig); } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java index 6976299cc7d..1775dbe53c1 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -43,13 +43,12 @@ public class VespaDocumentAccess extends DocumentAccess { private boolean shutDown = false; VespaDocumentAccess(DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig, String slobroksConfigId, MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); - this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)) + this.parameters = new MessageBusParams() .setDocumentProtocolPoliciesConfig(policiesConfig, distributionConfig); this.parameters.setDocumentmanagerConfig(documentmanagerConfig); this.parameters.getRPCNetworkParams().setSlobrokConfigId(slobroksConfigId); diff --git a/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java index c97e128b170..4d6f470637b 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; -import com.yahoo.compress.ZstdOuputStream; +import com.yahoo.compress.ZstdOutputStream; import com.yahoo.io.NativeIO; import com.yahoo.log.LogFileDb; import com.yahoo.protect.Process; @@ -401,7 +401,7 @@ class LogFileHandler <LOGTYPE> { Path compressedFile = Paths.get(oldFile.toString() + ".zst"); int bufferSize = 2*1024*1024; try (FileOutputStream fileOut = AtomicFileOutputStream.create(compressedFile); - ZstdOuputStream out = new ZstdOuputStream(fileOut, bufferSize); + ZstdOutputStream out = new ZstdOutputStream(fileOut, bufferSize); FileInputStream in = new FileInputStream(oldFile.toFile())) { pageFriendlyTransfer(nativeIO, out, fileOut.getFD(), in, bufferSize); out.flush(); diff --git a/container-core/src/main/java/com/yahoo/restapi/HttpURL.java b/container-core/src/main/java/com/yahoo/restapi/HttpURL.java new file mode 100644 index 00000000000..e890b0fe71a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/HttpURL.java @@ -0,0 +1,451 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import ai.vespa.validation.StringWrapper; +import com.yahoo.net.DomainName; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.OptionalInt; +import java.util.StringJoiner; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; + +import static ai.vespa.validation.Validation.require; +import static ai.vespa.validation.Validation.requireInRange; +import static java.net.URLDecoder.decode; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.unmodifiableMap; +import static java.util.Objects.requireNonNull; + +/** + * This is the best class for creating, manipulating and inspecting HTTP URLs, because: + * <ul> + * <li>It is more restrictive than {@link URI}, but with a richer construction API, reducing risk of blunder. + * <ul> + * <li>Scheme must be HTTP or HTTPS.</li> + * <li>Authority must be a {@link DomainName}, with an optional port.</li> + * <li>{@link Path} must be normalized at all times.</li> + * <li>Only {@link Query} is allowed, in addition to the above.</li> + * </ul> + * </li> + * <li> + * It contains all those helpful builder methods that {@link URI} has none of. + * <ul> + * <li>{@link Path} can be parsed, have segments or other paths appended, and cut.</li> + * <li>{@link Query} can be parsed, and keys and key-value pairs can be inserted or removed.</li> + * </ul> + * All these (except the parse methods) operate on <em>decoded</em> values. + * </li> + * <li>It makes it super-easy to use a {@link StringWrapper} for validation of path and query segments.</li> + * </ul> + * + * @author jonmv + */ +public class HttpURL { + + private final Scheme scheme; + private final DomainName domain; + private final int port; + private final Path path; + private final Query query; + + private HttpURL(Scheme scheme, DomainName domain, int port, Path path, Query query) { + this.scheme = requireNonNull(scheme); + this.domain = requireNonNull(domain); + this.port = requireInRange(port, "port number", -1, (1 << 16) - 1); + this.path = requireNonNull(path); + this.query = requireNonNull(query); + } + + public static HttpURL create(Scheme scheme, DomainName domain, int port, Path path, Query query) { + return new HttpURL(scheme, domain, port, path, query); + } + + public static HttpURL create(Scheme scheme, DomainName domain, int port, Path path) { + return create(scheme, domain, port, path, Query.empty()); + } + + public static HttpURL create(Scheme scheme, DomainName domain, int port) { + return create(scheme, domain, port, Path.empty(), Query.empty()); + } + + public static HttpURL create(Scheme scheme, DomainName domain) { + return create(scheme, domain, -1); + } + + public static HttpURL from(URI uri) { + return from(uri, HttpURL::requirePathSegment, HttpURL::requireNothing); + } + + public static HttpURL from(URI uri, Consumer<String> pathValidator, Consumer<String> queryValidator) { + if ( ! uri.normalize().equals(uri)) + throw new IllegalArgumentException("uri should be normalized, but got: " + uri); + + return create(Scheme.of(uri.getScheme()), + DomainName.of(requireNonNull(uri.getHost(), "URI must specify a host")), + uri.getPort(), + Path.parse(uri.getRawPath(), pathValidator), + Query.parse(uri.getRawQuery(), queryValidator)); + } + + public HttpURL withScheme(Scheme scheme) { + return create(scheme, domain, port, path, query); + } + + public HttpURL withDomain(DomainName domain) { + return create(scheme, domain, port, path, query); + } + + public HttpURL withPort(int port) { + return create(scheme, domain, port, path, query); + } + + public HttpURL withoutPort() { + return create(scheme, domain, -1, path, query); + } + + public HttpURL withPath(Path path) { + return create(scheme, domain, port, path, query); + } + + public HttpURL withQuery(Query query) { + return create(scheme, domain, port, path, query); + } + + public Scheme scheme() { + return scheme; + } + + public DomainName domain() { + return domain; + } + + public OptionalInt port() { + return port == -1 ? OptionalInt.empty() : OptionalInt.of(port); + } + + public Path path() { + return path; + } + + public Query query() { + return query; + } + + /** Returns an absolute, hierarchical URI representing this HTTP URL. */ + public URI asURI() { + try { + return new URI(scheme.name() + "://" + domain.value() + (port == -1 ? "" : ":" + port) + path.raw() + query.raw()); + } + catch (URISyntaxException e) { + throw new IllegalStateException("invalid URI, this should not happen", e); + } + } + + /** Require that the given string (possibly decoded multiple times) contains none of {@code '/', '?', '#'}, and isn't either of {@code "", ".", ".."}. */ + public static String requirePathSegment(String value) { + while ( ! value.equals(value = decode(value, UTF_8))); + require( ! value.contains("/"), value, "path segment decoded cannot contain '/'"); + require( ! value.contains("?"), value, "path segment decoded cannot contain '?'"); + require( ! value.contains("#"), value, "path segment decoded cannot contain '#'"); + return Path.requireNonNormalizable(value); + } + + private static void requireNothing(String value) { } + + public static class Path { + + private final List<String> segments; + private final boolean trailingSlash; + private final UnaryOperator<String> validator; + + private Path(List<String> segments, boolean trailingSlash, UnaryOperator<String> validator) { + this.segments = requireNonNull(segments); + this.trailingSlash = trailingSlash; + this.validator = requireNonNull(validator); + } + + /** Creates a new, empty path, with a trailing slash, using {@link HttpURL#requirePathSegment} for segment validation. */ + public static Path empty() { + return empty(HttpURL::requirePathSegment); + } + + /** Creates a new, empty path, with a trailing slash, using the indicated validator for segments. */ + public static Path empty(Consumer<String> validator) { + return new Path(List.of(), true, segmentValidator(validator)); + } + + /** Creates a new path with the given <em>decoded</em> segments. */ + public static Path from(List<String> segments) { + return from(segments, __ -> { }); + } + + /** Creates a new path with the given <em>decoded</em> segments, and the validator applied to each segment. */ + public static Path from(List<String> segments, Consumer<String> validator) { + return empty(validator).append(segments, true); + } + + /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative. */ + public static Path parse(String raw) { + return parse(raw, HttpURL::requirePathSegment); + } + + /** Parses the given raw, normalized path string; this ignores whether the path is absolute or relative.) */ + public static Path parse(String raw, Consumer<String> validator) { + Path base = new Path(List.of(), raw.endsWith("/"), segmentValidator(validator)); + if (raw.startsWith("/")) raw = raw.substring(1); + if (raw.isEmpty()) return base; + List<String> segments = new ArrayList<>(); + for (String segment : raw.split("/")) segments.add(decode(segment, UTF_8)); + if (segments.isEmpty()) requireNonNormalizable(""); // Raw path was only slashes. + return base.append(segments); + } + + private static UnaryOperator<String> segmentValidator(Consumer<String> validator) { + requireNonNull(validator, "segment validator cannot be null"); + return value -> { + requireNonNormalizable(value); + validator.accept(value); + return value; + }; + } + + private static String requireNonNormalizable(String segment) { + return require( ! (segment.isEmpty() || segment.equals(".") || segment.equals("..")), + segment, "path segments cannot be \"\", \".\", or \"..\""); + } + + /** Returns a copy of this where only the first segments are retained, and with a trailing slash. */ + public Path head(int count) { + return count == segments.size() ? this : new Path(segments.subList(0, count), true, validator); + } + + /** Returns a copy of this where only the last segments are retained. */ + public Path tail(int count) { + return count == segments.size() ? this : new Path(segments.subList(segments.size() - count, segments.size()), trailingSlash, validator); + } + + /** Returns a copy of this where the first segments are skipped. */ + public Path skip(int count) { + return count == 0 ? this : new Path(segments.subList(count, segments.size()), trailingSlash, validator); + } + + /** Returns a copy of this where the last segments are cut off, and with a trailing slash. */ + public Path cut(int count) { + return count == 0 ? this : new Path(segments.subList(0, segments.size() - count), true, validator); + } + + /** Returns a copy of this with the <em>decoded</em> segment appended at the end; it may not be either of {@code ""}, {@code "."} or {@code ".."}. */ + public Path append(String segment) { + return append(List.of(segment), trailingSlash); + } + + /** Returns a copy of this all segments of the other path appended, with a trailing slash as per the appendage. */ + public Path append(Path other) { + return append(other.segments, other.trailingSlash); + } + + /** Returns a copy of this all given segments appended, with a trailing slash as per this path. */ + public Path append(List<String> segments) { + return append(segments, trailingSlash); + } + + private Path append(List<String> segments, boolean trailingSlash) { + List<String> copy = new ArrayList<>(this.segments); + for (String segment : segments) copy.add(validator.apply(segment)); + return new Path(copy, trailingSlash, validator); + } + + /** Whether this path has a trailing slash. */ + public boolean hasTrailingSlash() { + return trailingSlash; + } + + /** Returns a copy of this which encodes a trailing slash. */ + public Path withTrailingSlash() { + return new Path(segments, true, validator); + } + + /** Returns a copy of this which does not encode a trailing slash. */ + public Path withoutTrailingSlash() { + return new Path(segments, false, validator); + } + + /** The <em>URL decoded</em> segments that make up this path; never {@code null}, {@code ""}, {@code "."} or {@code ".."}. */ + public List<String> segments() { + return Collections.unmodifiableList(segments); + } + + /** A raw path string which parses to this, by splitting on {@code "/"}, and then URL decoding. */ + String raw() { + StringJoiner joiner = new StringJoiner("/", "/", trailingSlash ? "/" : "").setEmptyValue(trailingSlash ? "/" : ""); + for (String segment : segments) joiner.add(encode(segment, UTF_8)); + return joiner.toString(); + } + + /** Intentionally not usable for constructing new URIs. Use {@link HttpURL} for that instead. */ + @Override + public String toString() { + return "path '" + raw() + "'"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Path path = (Path) o; + return trailingSlash == path.trailingSlash && segments.equals(path.segments); + } + + @Override + public int hashCode() { + return Objects.hash(segments, trailingSlash); + } + + } + + + public static class Query { + + private final Map<String, String> values; + private final UnaryOperator<String> validator; + + private Query(Map<String, String> values, UnaryOperator<String> validator) { + this.values = requireNonNull(values); + this.validator = requireNonNull(validator); + } + + /** Creates a new, empty query part. */ + public static Query empty() { + return empty(__ -> { }); + } + + /** Creates a new, empty query part, using the indicated string wrapper for keys and non-null values. */ + public static Query empty(Consumer<String> validator) { + return new Query(Map.of(), entryValidator(validator)); + } + + /** Creates a new query part with the given <em>decoded</em> values. */ + public static Query from(Map<String, String> values) { + return from(values, __ -> { }); + } + + /** Creates a new query part with the given <em>decoded</em> values, and the validator applied to each pair. */ + public static Query from(Map<String, String> values, Consumer<String> validator) { + return empty(validator).merge(values); + } + + /** Parses the given raw query string. */ + public static Query parse(String raw) { + return parse(raw, __-> { }); + } + /** Parses the given raw query string, using the indicated string wrapper to hold keys and non-null values. */ + public static Query parse(String raw, Consumer<String> validator) { + if (raw == null) return empty(validator); + Map<String, String> values = new LinkedHashMap<>(); + for (String pair : raw.split("&")) { + int split = pair.indexOf("="); + String key, value; + if (split == -1) { key = pair; value = null; } + else { key = pair.substring(0, split); value = pair.substring(split + 1); } + values.put(decode(key, UTF_8), value == null ? null : decode(value, UTF_8)); + } + return empty(validator).merge(values); + } + + private static UnaryOperator<String> entryValidator(Consumer<String> validator) { + requireNonNull(validator); + return value -> { + validator.accept(value); + return value; + }; + } + + /** Returns a copy of this with the <em>decoded</em> non-null key pointing to the <em>decoded</em> non-null value. */ + public Query put(String key, String value) { + Map<String, String> copy = new LinkedHashMap<>(values); + copy.put(validator.apply(requireNonNull(key)), validator.apply(requireNonNull(value))); + return new Query(copy, validator); + } + + /** Returns a copy of this with the <em>decoded</em> non-null key pointing to "nothing". */ + public Query add(String key) { + Map<String, String> copy = new LinkedHashMap<>(values); + copy.put(validator.apply(requireNonNull(key)), null); + return new Query(copy, validator); + } + + /** Returns a copy of this without any key-value pair with the <em>decoded</em> key. */ + public Query remove(String key) { + Map<String, String> copy = new LinkedHashMap<>(values); + copy.remove(validator.apply(requireNonNull(key))); + return new Query(copy, validator); + } + + /** Returns a copy of this with all mappings from the other query added to this, possibly overwriting existing mappings. */ + public Query merge(Query other) { + return merge(other.values); + } + + /** Returns a copy of this with all given mappings added to this, possibly overwriting existing mappings. */ + public Query merge(Map<String, String> values) { + Map<String, String> copy = new LinkedHashMap<>(this.values); + values.forEach((key, value) -> copy.put(validator.apply(requireNonNull(key, "keys cannot be null")), + value == null ? null : validator.apply(value))); + return new Query(copy, validator); + } + + /** The <em>URL decoded</em> key-value pairs that make up this query; keys and values may be {@code ""}, and values are {@code null} when only key was specified. */ + public Map<String, String> entries() { + return unmodifiableMap(values); + } + + /** A raw query string, with {@code '?'} prepended, that parses to this, by splitting on {@code "&"}, then on {@code "="}, and then URL decoding; or the empty string if this is empty. */ + private String raw() { + StringJoiner joiner = new StringJoiner("&", "?", "").setEmptyValue(""); + values.forEach((key, value) -> joiner.add(encode(key, UTF_8) + + (value == null ? "" : "=" + encode(value, UTF_8)))); + return joiner.toString(); + } + + /** Intentionally not usable for constructing new URIs. Use {@link HttpURL} for that instead. */ + @Override + public String toString() { + return "query '" + raw() + "'"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Query query = (Query) o; + return values.equals(query.values); + } + + @Override + public int hashCode() { + return Objects.hash(values); + } + + } + + + public enum Scheme { + http, + https; + public static Scheme of(String scheme) { + if (scheme.equalsIgnoreCase(http.name())) return http; + if (scheme.equalsIgnoreCase(https.name())) return https; + throw new IllegalArgumentException("scheme must be HTTP or HTTPS"); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java index b96488c6781..c639432db89 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Path.java +++ b/container-core/src/main/java/com/yahoo/restapi/Path.java @@ -2,14 +2,10 @@ package com.yahoo.restapi; import java.net.URI; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Stream; +import java.util.function.Consumer; /** * A normalized path which is able to match strings containing bracketed placeholders and return the @@ -32,50 +28,47 @@ import java.util.stream.Stream; public class Path { // This path - private final String pathString; - private final String[] elements; + private final HttpURL.Path path; // Info about the last match private final Map<String, String> values = new HashMap<>(); - private String rest = ""; + private HttpURL.Path rest; + /** Creates a new Path for matching the given URI against patterns, which uses {@link HttpURL#requirePathSegment} as a segment validator. */ public Path(URI uri) { - this.pathString = requireNormalized(uri).getRawPath(); - this.elements = splitAbsolutePath(pathString, (part) -> URLDecoder.decode(part, StandardCharsets.UTF_8)); + this.path = HttpURL.Path.parse(uri.getRawPath()); + } + + /** Creates a new Path for matching the given URI against patterns, with the given path segment validator. */ + public Path(URI uri, Consumer<String> validator) { + this.path = HttpURL.Path.parse(uri.getRawPath(), validator); } private boolean matchesInner(String pathSpec) { values.clear(); - String[] specElements = splitAbsolutePath(pathSpec, Function.identity()); + List<String> specElements = HttpURL.Path.parse(pathSpec).segments(); boolean matchPrefix = false; - if (specElements.length > 1 && specElements[specElements.length-1].equals("{*}")) { + if (specElements.size() > 1 && specElements.get(specElements.size() - 1).equals("{*}")) { matchPrefix = true; - specElements = Arrays.copyOf(specElements, specElements.length-1); + specElements = specElements.subList(0, specElements.size() - 1); } if (matchPrefix) { - if (this.elements.length < specElements.length) return false; + if (path.segments().size() < specElements.size()) return false; } else { // match exact - if (this.elements.length != specElements.length) return false; + if (path.segments().size() != specElements.size()) return false; } - for (int i = 0; i < specElements.length; i++) { - if (specElements[i].startsWith("{") && specElements[i].endsWith("}")) // placeholder - values.put(specElements[i].substring(1, specElements[i].length()-1), elements[i]); - else if ( ! specElements[i].equals(this.elements[i])) + for (int i = 0; i < specElements.size(); i++) { + if (specElements.get(i).startsWith("{") && specElements.get(i).endsWith("}")) // placeholder + values.put(specElements.get(i).substring(1, specElements.get(i).length() - 1), path.segments().get(i)); + else if ( ! specElements.get(i).equals(path.segments().get(i))) return false; } - - if (matchPrefix) { - StringBuilder rest = new StringBuilder(); - for (int i = specElements.length; i < this.elements.length; i++) - rest.append(elements[i]).append("/"); - if ( ! pathString.endsWith("/") && rest.length() > 0) - rest.setLength(rest.length() - 1); - this.rest = rest.toString(); - } - + + rest = matchPrefix ? path.skip(specElements.size()) : null; + return true; } @@ -104,34 +97,15 @@ public class Path { } /** - * Returns the rest of the last matched path. - * This is always the empty string (never null) unless the path spec ends with {*} + * Returns the rest of the last matched path, or {@code null} if the path spec didn't end with {*}. */ - public String getRest() { return rest; } - - public String asString() { - return pathString; + public HttpURL.Path getRest() { + return rest; } @Override public String toString() { - return "path '" + String.join("/", elements) + "'"; - } - - private static URI requireNormalized(URI uri) { - Objects.requireNonNull(uri); - if (!uri.normalize().equals(uri)) throw new IllegalArgumentException("Expected normalized URI, got '" + uri + "'"); - return uri; + return path.toString(); } - private static String[] splitAbsolutePath(String path, Function<String, String> partParser) { - String[] parts = Stream.of(path.split("/")) - .map(partParser) - .toArray(String[]::new); - for (var part : parts) { - if (part.equals("..")) throw new IllegalArgumentException("Expected absolute path, got '" + path + "'"); - } - return parts; - } - } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java index d6cc0cccf3f..353ac3eb5cc 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApi.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -153,7 +153,9 @@ public interface RestApi { default double getDoubleOrThrow(String name) { return Double.parseDouble(getStringOrThrow(name)); } } - interface PathParameters extends Parameters {} + interface PathParameters extends Parameters { + Optional<HttpURL.Path> getRest(); + } interface QueryParameters extends Parameters { List<String> getStringList(String name); } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index 09de7ffa133..cc243a3e92b 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -429,16 +429,16 @@ class RestApiImpl implements RestApi { private class PathParametersImpl implements RestApi.RequestContext.PathParameters { @Override public Optional<String> getString(String name) { - if (name.equals("*")) { - String rest = pathMatcher.getRest(); - return rest.isEmpty() ? Optional.empty() : Optional.of(rest); - } return Optional.ofNullable(pathMatcher.get(name)); } @Override public String getStringOrThrow(String name) { return getString(name) .orElseThrow(() -> new RestApiException.BadRequest("Path parameter '" + name + "' is missing")); } + @Override public Optional<HttpURL.Path> getRest() { + return Optional.ofNullable(pathMatcher.getRest()); + } + } private class QueryParametersImpl implements RestApi.RequestContext.QueryParameters { diff --git a/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java b/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java new file mode 100644 index 00000000000..858513c2a69 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/restapi/HttpURLTest.java @@ -0,0 +1,202 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import ai.vespa.validation.Name; +import com.yahoo.net.DomainName; +import com.yahoo.restapi.HttpURL.Query; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.function.Consumer; + +import static com.yahoo.net.DomainName.localhost; +import static com.yahoo.restapi.HttpURL.Scheme.http; +import static com.yahoo.restapi.HttpURL.Scheme.https; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author jonmv + */ +class HttpURLTest { + + @Test + void testConversionBackAndForth() { + for (String uri : List.of("http://minimal", + "http://empty.query?", + "http://zero-port:0?no=path", + "http://only-path/", + "https://strange/queries?=&foo", + "https://weirdness?=foo", + "https://encoded/%3F%3D%26%2F?%3F%3D%26%2F=%3F%3D%26%2F", + "https://host.at.domain:123/one/two/?three=four&five")) { + Consumer<String> pathValidator = __ -> { }; + assertEquals(uri, HttpURL.from(URI.create(uri), pathValidator, pathValidator).asURI().toString(), + "uri '" + uri + "' should be returned unchanged"); + } + } + + @Test + void testModification() { + HttpURL url = HttpURL.create(http, localhost).withPath(HttpURL.Path.empty(Name::of)); + assertEquals(http, url.scheme()); + assertEquals(localhost, url.domain()); + assertEquals(OptionalInt.empty(), url.port()); + assertEquals(HttpURL.Path.empty(Name::of), url.path()); + assertEquals(HttpURL.Query.empty(Name::of), url.query()); + + url = url.withScheme(https) + .withDomain(DomainName.of("domain")) + .withPort(0) + .withPath(url.path().append("foo").withoutTrailingSlash()) + .withQuery(url.query().put("boo", "bar").add("baz")); + assertEquals(https, url.scheme()); + assertEquals(DomainName.of("domain"), url.domain()); + assertEquals(OptionalInt.of(0), url.port()); + assertEquals(HttpURL.Path.parse("/foo", Name::of), url.path()); + assertEquals(HttpURL.Query.parse("boo=bar&baz", Name::of), url.query()); + } + + @Test + void testInvalidURIs() { + assertEquals("scheme must be HTTP or HTTPS", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("file:/txt"))).getMessage()); + + assertEquals("URI must specify a host", + assertThrows(NullPointerException.class, + () -> HttpURL.from(URI.create("http:///foo"))).getMessage()); + + assertEquals("port number must be at least '-1' and at most '65535', but got: '65536'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo:65536/bar"))).getMessage()); + + assertEquals("uri should be normalized, but got: http://foo//", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo//"))).getMessage()); + + assertEquals("uri should be normalized, but got: http://foo/./", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/./"))).getMessage()); + + assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/.."))).getMessage()); + + assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/.%2E"))).getMessage()); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: '/'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo/%2F"), Name::of, Name::of)).getMessage()); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: '/'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo?%2F"), Name::of, Name::of)).getMessage()); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: ''", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.from(URI.create("http://foo?"), Name::of, Name::of)).getMessage()); + } + + @Test + void testPath() { + HttpURL.Path path = HttpURL.Path.parse("foo/bar/baz", Name::of); + List<String> expected = List.of("foo", "bar", "baz"); + assertEquals(expected, path.segments()); + + assertEquals(expected.subList(1, 3), path.skip(1).segments()); + assertEquals(expected.subList(0, 2), path.cut(1).segments()); + assertEquals(expected.subList(1, 2), path.skip(1).cut(1).segments()); + + assertEquals("path '/foo/bar/baz/'", path.withTrailingSlash().toString()); + assertEquals(path, path.withoutTrailingSlash().withoutTrailingSlash()); + + assertEquals(List.of("one", "foo", "bar", "baz", "two"), + HttpURL.Path.from(List.of("one")).append(path).append("two").segments()); + + assertEquals(List.of(expected.get(2), expected.get(0)), + path.append(path).cut(2).skip(2).segments()); + + for (int i = 0; i < 3; i++) { + assertEquals(path.head(i), path.cut(3 - i)); + assertEquals(path.tail(i), path.skip(3 - i)); + } + + assertThrows(NullPointerException.class, + () -> path.append((String) null)); + + List<String> names = new ArrayList<>(); + names.add(null); + assertThrows(NullPointerException.class, + () -> path.append(names)); + + assertEquals("name must match '[A-Za-z][A-Za-z0-9_-]{0,63}', but got: '???'", + assertThrows(IllegalArgumentException.class, + () -> path.append("???")).getMessage()); + + assertEquals("fromIndex(2) > toIndex(1)", + assertThrows(IllegalArgumentException.class, + () -> path.cut(2).skip(2)).getMessage()); + + assertEquals("path segment decoded cannot contain '/', but got: '/'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.Path.empty().append("%2525252525252525%2525252525253%25252532%252525%252534%36")).getMessage()); + + assertEquals("path segment decoded cannot contain '?', but got: '?'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.Path.empty().append("?")).getMessage()); + + assertEquals("path segment decoded cannot contain '#', but got: '#'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.Path.empty().append("#")).getMessage()); + + assertEquals("path segments cannot be \"\", \".\", or \"..\", but got: '..'", + assertThrows(IllegalArgumentException.class, + () -> HttpURL.Path.empty().append("%2E%25252E")).getMessage()); + } + + @Test + void testQuery() { + Query query = Query.parse("foo=bar&baz", Name::of); + Map<String, String> expected = new LinkedHashMap<>(); + expected.put("foo", "bar"); + expected.put("baz", null); + assertEquals(expected, query.entries()); + + expected.remove("baz"); + assertEquals(expected, query.remove("baz").entries()); + + expected.put("baz", null); + expected.remove("foo"); + assertEquals(expected, query.remove("foo").entries()); + assertEquals(expected, Query.empty(Name::of).add("baz").entries()); + + assertEquals("query '?foo=bar&baz=bax&quu=fez&moo'", + query.put("baz", "bax").merge(Query.from(Map.of("quu", "fez"))).add("moo").toString()); + + assertThrows(NullPointerException.class, + () -> query.remove(null)); + + assertThrows(NullPointerException.class, + () -> query.add(null)); + + assertThrows(NullPointerException.class, + () -> query.put(null, "hax")); + + assertThrows(NullPointerException.class, + () -> query.put("hax", null)); + + Map<String, String> names = new LinkedHashMap<>(); + names.put(null, "hax"); + assertThrows(NullPointerException.class, + () -> query.merge(names)); + } + +} diff --git a/container-core/src/test/java/com/yahoo/restapi/PathTest.java b/container-core/src/test/java/com/yahoo/restapi/PathTest.java index 5cbf80ff2ad..4786eb9775c 100644 --- a/container-core/src/test/java/com/yahoo/restapi/PathTest.java +++ b/container-core/src/test/java/com/yahoo/restapi/PathTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.net.URI; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -34,7 +35,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("", path.getRest()); + assertEquals("/", path.getRest().raw()); } { @@ -42,7 +43,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("kanoo", path.getRest()); + assertEquals("/kanoo", path.getRest().raw()); } { @@ -50,7 +51,7 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("kanoo/trips", path.getRest()); + assertEquals("/kanoo/trips", path.getRest().raw()); } { @@ -58,17 +59,19 @@ public class PathTest { assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); - assertEquals("kanoo/trips/", path.getRest()); + assertEquals("/kanoo/trips/", path.getRest().raw()); } } @Test public void testUrlEncodedPath() { assertTrue(new Path(URI.create("/a/%62/c")).matches("/a/b/c")); - assertFalse(new Path(URI.create("/a/b%2fc")).matches("/a/b/c")); - assertFalse(new Path(URI.create("/foo")).matches("/foo/bar/%2e%2e")); + assertFalse(new Path(URI.create("/a/b%2fc"), __ -> { }).matches("/a/b/c")); + assertThrows("path segments cannot be \"\", \".\", or \"..\", but got: '..'", + IllegalArgumentException.class, + () -> new Path(URI.create("/foo")).matches("/foo/bar/%2e%2e")); - Path path = new Path(URI.create("/%61/%2f/%63")); + Path path = new Path(URI.create("/%61/%2f/%63"), __ -> { }); assertTrue(path.matches("/a/{slash}/{c}")); assertEquals("/", path.get("slash")); assertEquals("c", path.get("c")); diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 034081f4620..6268e1e6fb4 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -121,6 +121,10 @@ <groupId>io.airlift</groupId> <artifactId>aircompressor</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 46dcaf17abc..91a3181cb68 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -69,14 +69,18 @@ public final class SessionCache extends AbstractComponent { @Inject public SessionCache(NetworkMultiplexerProvider nets, ContainerMbusConfig containerMbusConfig, DocumentTypeManager documentTypeManager, - LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, + MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(nets::net, containerMbusConfig, documentTypeManager, - loadTypeConfig, messagebusConfig, policiesConfig, distributionConfig); + null/*TODO: Remove on Vespa 8*/, messagebusConfig, policiesConfig, distributionConfig); } + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public SessionCache(Supplier<NetworkMultiplexer> net, ContainerMbusConfig containerMbusConfig, DocumentTypeManager documentTypeManager, LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, @@ -86,7 +90,6 @@ public final class SessionCache extends AbstractComponent { containerMbusConfig, messagebusConfig, new DocumentProtocol(documentTypeManager, - new LoadTypeSet(loadTypeConfig), policiesConfig, distributionConfig)); } diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java index c509fb917fa..b9f33506894 100644 --- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java +++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java @@ -36,6 +36,7 @@ public class MbusClientProviderTest { testClient(new SessionConfig(builder)); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void testClient(SessionConfig config) { SessionCache cache = new SessionCache(() -> NetworkMultiplexer.dedicated(new NullNetwork()), new ContainerMbusConfig.Builder().build(), diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java index b2b9dd47514..bbce02ed97a 100644 --- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java +++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java @@ -28,6 +28,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.util.logging.Level; @@ -71,24 +72,23 @@ public class GUIHandler extends ThreadedHttpRequestHandler { if (path.matches("/querybuilder/")) { return new FileResponse("_includes/index.html", null, null); } - if (!path.matches("/querybuilder/{*}") ) { + if ( ! path.matches("/querybuilder/{*}") ) { return ErrorResponse.notFoundError("Nothing at path:" + path); } - String filepath = path.getRest(); - if (!isValidPath(filepath) && !filepath.equals("config.json")){ + String filepath = String.join("/", path.getRest().segments()); + if ( ! filepath.equals("config.json") && ! isValidPath(filepath)){ return ErrorResponse.notFoundError("Nothing at path:" + filepath); } return new FileResponse(filepath, indexModel, rankProfilesConfig); } private static boolean isValidPath(String path) { - InputStream in = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+path); - boolean isValid = (in != null); - if(isValid){ - try { in.close(); } catch (IOException e) {/* Problem with closing inputstream */} + InputStream in = GUIHandler.class.getClassLoader().getResourceAsStream("gui/" + path); + if (in != null){ + try { in.close(); } catch (IOException e) { throw new UncheckedIOException(e); } + return true; } - - return isValid; + return false; } private static class FileResponse extends HttpResponse { @@ -112,12 +112,12 @@ public class GUIHandler extends ThreadedHttpRequestHandler { String json = "{}"; try { json = getGUIConfig(); } catch (IOException e) { /*Something happened while parsing JSON */ } is = new ByteArrayInputStream(json.getBytes()); - } else{ - is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+this.path); + } else { + is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/" + this.path); } byte[] buf = new byte[1024]; int numRead; - while ( (numRead = is.read(buf) ) >= 0) { + while ((numRead = is.read(buf)) >= 0) { out.write(buf, 0, numRead); } } @@ -152,8 +152,6 @@ public class GUIHandler extends ThreadedHttpRequestHandler { return "image/x-icon"; } else if (path.endsWith(".json")) { return "application/json"; - } else if (path.endsWith(".ttf")) { - return "font/ttf"; } return "text/html"; } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 489214bebf8..71a93607b4b 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -90,6 +90,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { } @Override + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public LoadTypeSet getLoadTypeSet() { return ((MessageBusDocumentAccess) access.delegate()).getParams().getLoadTypes(); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index b2e4821f164..9330e43eaf7 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -76,6 +76,12 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public interface VisitorSessionFactory { VisitorSession createVisitorSession(VisitorParameters params) throws ParseException; + + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 LoadTypeSet getLoadTypeSet(); } @@ -119,6 +125,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { return query.properties().getString(streamingSelection); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void setVisitorParameters(String searchCluster, Route route, String documentType) { params.setDocumentSelection(createSelectionString(documentType, createQuerySelectionString())); params.setTimeoutMs(query.getTimeout()); // Per bucket visitor timeout @@ -134,6 +141,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.visitInconsistentBuckets(true); params.setPriority(DocumentProtocol.Priority.VERY_HIGH); + // TODO remove on Vespa 8 if (query.properties().getString(streamingLoadtype) != null) { LoadType loadType = visitorSessionFactory.getLoadTypeSet().getNameMap().get(query.properties().getString(streamingLoadtype)); if (loadType != null) { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index 1d07cafeda9..b1bc926daed 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -31,8 +31,9 @@ import static org.junit.Assert.*; /** * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VdsVisitorTestCase { - private LoadTypeSet loadTypeSet = new LoadTypeSet(); + private LoadTypeSet loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 public VdsVisitorTestCase() { loadTypeSet.addLoadType(1, "low", DocumentProtocol.Priority.LOW_1); @@ -489,7 +490,7 @@ public class VdsVisitorTestCase { private static class MockVisitorSessionFactory implements VdsVisitor.VisitorSessionFactory { private VisitorParameters params; - private LoadTypeSet loadTypeSet; + private LoadTypeSet loadTypeSet; // TODO remove on Vespa 8 private boolean timeoutQuery = false; private boolean failQuery = false; @@ -504,6 +505,7 @@ public class VdsVisitorTestCase { } @Override + // TODO: Remove on Vespa 8 public LoadTypeSet getLoadTypeSet() { return loadTypeSet; } diff --git a/container-test/pom.xml b/container-test/pom.xml index 7c739faad26..d4ea39fa3c9 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -92,5 +92,10 @@ <artifactId>aircompressor</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <scope>compile</scope> + </dependency> </dependencies> </project> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java index 1788154b9e2..01488711f59 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -187,6 +188,7 @@ public class Bill { private BigDecimal cpuCost; private BigDecimal memoryCost; private BigDecimal diskCost; + private NodeResources.Architecture architecture; public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt) { this.id = id; @@ -198,7 +200,7 @@ public class Bill { } public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt, ZonedDateTime startedAt, ZonedDateTime endedAt, ApplicationId applicationId, ZoneId zoneId, - BigDecimal cpuHours, BigDecimal memoryHours, BigDecimal diskHours, BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost) { + BigDecimal cpuHours, BigDecimal memoryHours, BigDecimal diskHours, BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost, NodeResources.Architecture architecture) { this(id, description, amount, plan, agent, addedAt); this.startedAt = startedAt; this.endedAt = endedAt; @@ -214,6 +216,7 @@ public class Bill { this.cpuCost = cpuCost; this.memoryCost = memoryCost; this.diskCost = diskCost; + this.architecture = architecture; } /** The opaque ID of this */ @@ -290,6 +293,10 @@ public class Bill { return Optional.ofNullable(diskCost); } + public Optional<NodeResources.Architecture> getArchitecture() { + return Optional.ofNullable(architecture); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index 61f8844482c..43312f9332c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -109,9 +109,6 @@ public interface BillingController { /** Get all bills from the system */ List<Bill> getBills(); - /** Delete billing contact information from the tenant */ - void deleteBillingInfo(TenantName tenant, Set<User> users, boolean isPrivileged); - /** Get the bill collection method for the given tenant */ default CollectionMethod getCollectionMethod(TenantName tenant) { return CollectionMethod.NONE; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index f4d3577aeec..295c993f409 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -162,9 +162,6 @@ public class MockBillingController implements BillingController { } @Override - public void deleteBillingInfo(TenantName tenant, Set<User> users, boolean isPrivileged) {} - - @Override public CollectionMethod getCollectionMethod(TenantName tenant) { return collectionMethod.getOrDefault(tenant, CollectionMethod.AUTO); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java index 5fb4d853e67..4757fa76224 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java @@ -117,7 +117,8 @@ public class PlanRegistryMock implements PlanRegistry { usage.getDiskMillis().divide(millisPerHour, RoundingMode.HALF_UP), cpuCost, memCost, - dgbCost + dgbCost, + usage.getArchitecture() ); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index b9b7881745a..42368fa358d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.net.DomainName; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -52,9 +54,9 @@ public interface ConfigServer { ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); - Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath); + Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, Path restPath); - String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath); + String getServiceStatusPage(DeploymentId deployment, String serviceName, DomainName node, Path subPath); /** * Gets the Vespa logs of the given deployment. @@ -73,7 +75,7 @@ public interface ConfigServer { * @param path path within package to get * @param requestUri request URI on the controller, used to rewrite paths in response from config server */ - ProxyResponse getApplicationPackageContent(DeploymentId deployment, String path, URI requestUri); + ProxyResponse getApplicationPackageContent(DeploymentId deployment, Path path, URI requestUri); List<ClusterMetrics> getDeploymentMetrics(DeploymentId deployment); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java index fe7beb538da..68ebc5e86aa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java @@ -20,6 +20,6 @@ public class NodeRepoStats { public Load load() { return load; } public Load activeLoad() { return activeLoad; } - public List<ApplicationStats> applicationStats() { return applicationStats; } + public List<ApplicationStats> applicationStats() { return applicationStats; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MailerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MailerException.java new file mode 100644 index 00000000000..0febc296fc8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MailerException.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +/** + * MailerException wrap all possible Mailer implementation exceptions + * + * @author enygaard + */ +public class MailerException extends RuntimeException { + + public MailerException(Throwable ex) { + super(ex); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java index 3bc9580307b..c756100e563 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import java.math.BigDecimal; @@ -19,11 +20,12 @@ public class CostInfo { private final BigDecimal cpuCost; private final BigDecimal memoryCost; private final BigDecimal diskCost; + private final NodeResources.Architecture architecture; public CostInfo(ApplicationId applicationId, ZoneId zoneId, BigDecimal cpuHours, BigDecimal memoryHours, BigDecimal diskHours, - BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost) { + BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost, NodeResources.Architecture architecture) { this.applicationId = applicationId; this.zoneId = zoneId; this.cpuHours = cpuHours; @@ -32,6 +34,7 @@ public class CostInfo { this.cpuCost = cpuCost; this.memoryCost = memoryCost; this.diskCost = diskCost; + this.architecture = architecture; } public ApplicationId getApplicationId() { @@ -70,4 +73,8 @@ public class CostInfo { return cpuCost.add(memoryCost).add(diskCost); } + public NodeResources.Architecture getArchitecture() { + return architecture; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java index 767ba4aa34b..944a5eaf696 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java @@ -17,8 +17,6 @@ public interface MeteringClient { void consume(Collection<ResourceSnapshot> resources); - MeteringData getMeteringData(TenantName tenantName, ApplicationName applicationName); - List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth); void refresh(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java index 742e3a01171..8191540e898 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceAllocation.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; +import com.yahoo.config.provision.NodeResources; + import java.util.Objects; /** @@ -10,16 +12,18 @@ import java.util.Objects; */ public class ResourceAllocation { - public static final ResourceAllocation ZERO = new ResourceAllocation(0, 0, 0); + public static final ResourceAllocation ZERO = new ResourceAllocation(0, 0, 0, NodeResources.Architecture.getDefault()); private final double cpuCores; private final double memoryGb; private final double diskGb; + private final NodeResources.Architecture architecture; - public ResourceAllocation(double cpuCores, double memoryGb, double diskGb) { + public ResourceAllocation(double cpuCores, double memoryGb, double diskGb, NodeResources.Architecture architecture) { this.cpuCores = cpuCores; this.memoryGb = memoryGb; this.diskGb = diskGb; + this.architecture = architecture; } public double usageFraction(ResourceAllocation total) { @@ -38,14 +42,18 @@ public class ResourceAllocation { return diskGb; } + public NodeResources.Architecture getArchitecture() { + return architecture; + } + /** Returns a copy of this with the given allocation added */ public ResourceAllocation plus(ResourceAllocation allocation) { - return new ResourceAllocation(cpuCores + allocation.cpuCores, memoryGb + allocation.memoryGb, diskGb + allocation.diskGb); + return new ResourceAllocation(cpuCores + allocation.cpuCores, memoryGb + allocation.memoryGb, diskGb + allocation.diskGb, architecture); } /** Returns a copy of this with each resource multiplied by given factor */ public ResourceAllocation multiply(double multiplicand) { - return new ResourceAllocation(cpuCores * multiplicand, memoryGb * multiplicand, diskGb * multiplicand); + return new ResourceAllocation(cpuCores * multiplicand, memoryGb * multiplicand, diskGb * multiplicand, architecture); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java index 2f277193231..8ae12c0e7ac 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClient.java @@ -18,8 +18,6 @@ public interface ResourceDatabaseClient { void writeResourceSnapshots(Collection<ResourceSnapshot> snapshots); - List<ResourceSnapshot> getResourceSnapshotsForMonth(TenantName tenantName, ApplicationName applicationName, YearMonth month); - List<ResourceUsage> getResourceSnapshotsForPeriod(TenantName tenantName, long startTimestamp, long endTimestamp); void refreshMaterializedView(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java index 5a4d250ea9d..c680990e240 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java @@ -45,19 +45,6 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { } @Override - public List<ResourceSnapshot> getResourceSnapshotsForMonth(TenantName tenantName, ApplicationName applicationName, YearMonth month) { - return resourceSnapshots.stream() - .filter(resourceSnapshot -> { - LocalDate snapshotDate = LocalDate.ofInstant(resourceSnapshot.getTimestamp(), ZoneId.of("UTC")); - return YearMonth.from(snapshotDate).equals(month) && - snapshotDate.getYear() == month.getYear() && - resourceSnapshot.getApplicationId().tenant().equals(tenantName) && - resourceSnapshot.getApplicationId().application().equals(applicationName); - }) - .collect(Collectors.toList()); - } - - @Override public Set<YearMonth> getMonthsWithSnapshotsForTenant(TenantName tenantName) { return Collections.emptySet(); } @@ -88,6 +75,7 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { a.getApplicationId(), a.getZoneId(), plan, + a.getArchitecture(), BigDecimal.valueOf(a.getCpuCores()).multiply(d), BigDecimal.valueOf(a.getMemoryGb()).multiply(d), BigDecimal.valueOf(a.getDiskGb()).multiply(d) @@ -100,10 +88,12 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { assert a.getApplicationId().equals(b.getApplicationId()); assert a.getZoneId().equals(b.getZoneId()); assert a.getPlan().equals(b.getPlan()); + assert a.getArchitecture().equals(b.getArchitecture()); return new ResourceUsage( a.getApplicationId(), a.getZoneId(), a.getPlan(), + a.getArchitecture(), a.getCpuMillis().add(b.getCpuMillis()), a.getMemoryMillis().add(b.getMemoryMillis()), a.getDiskMillis().add(b.getDiskMillis()) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java index 9b3de004bae..85ee23f4df0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java @@ -24,15 +24,15 @@ public class ResourceSnapshot { private final Instant timestamp; private final ZoneId zoneId; - public ResourceSnapshot(ApplicationId applicationId, double cpuCores, double memoryGb, double diskGb, Instant timestamp, ZoneId zoneId) { + public ResourceSnapshot(ApplicationId applicationId, double cpuCores, double memoryGb, double diskGb, NodeResources.Architecture architecture, Instant timestamp, ZoneId zoneId) { this.applicationId = applicationId; - this.resourceAllocation = new ResourceAllocation(cpuCores, memoryGb, diskGb); + this.resourceAllocation = new ResourceAllocation(cpuCores, memoryGb, diskGb, architecture); this.timestamp = timestamp; this.zoneId = zoneId; } - public static ResourceSnapshot from(ApplicationId applicationId, int nodes, double cpuCores, double memoryGb, double diskGb, Instant timestamp, ZoneId zoneId) { - return new ResourceSnapshot(applicationId, cpuCores * nodes, memoryGb * nodes, diskGb * nodes, timestamp, zoneId); + public static ResourceSnapshot from(ApplicationId applicationId, int nodes, double cpuCores, double memoryGb, double diskGb, NodeResources.Architecture architecture, Instant timestamp, ZoneId zoneId) { + return new ResourceSnapshot(applicationId, cpuCores * nodes, memoryGb * nodes, diskGb * nodes, architecture, timestamp, zoneId); } public static ResourceSnapshot from(List<Node> nodes, Instant timestamp, ZoneId zoneId) { @@ -48,6 +48,7 @@ public class ResourceSnapshot { nodes.stream().map(Node::resources).mapToDouble(NodeResources::vcpu).sum(), nodes.stream().map(Node::resources).mapToDouble(NodeResources::memoryGb).sum(), nodes.stream().map(Node::resources).mapToDouble(NodeResources::diskGb).sum(), + nodes.stream().map(node -> node.resources().architecture()).findFirst().orElse(NodeResources.Architecture.getDefault()), timestamp, zoneId ); @@ -81,6 +82,10 @@ public class ResourceSnapshot { return zoneId; } + public NodeResources.Architecture getArchitecture() { + return resourceAllocation.getArchitecture(); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java index fdea3b26372..850ca040d03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; @@ -15,8 +16,9 @@ public class ResourceUsage { private final BigDecimal cpuMillis; private final BigDecimal memoryMillis; private final BigDecimal diskMillis; + private final NodeResources.Architecture architecture; - public ResourceUsage(ApplicationId applicationId, ZoneId zoneId, Plan plan, + public ResourceUsage(ApplicationId applicationId, ZoneId zoneId, Plan plan, NodeResources.Architecture architecture, BigDecimal cpuMillis, BigDecimal memoryMillis, BigDecimal diskMillis) { this.applicationId = applicationId; this.zoneId = zoneId; @@ -24,6 +26,7 @@ public class ResourceUsage { this.memoryMillis = memoryMillis; this.diskMillis = diskMillis; this.plan = plan; + this.architecture = architecture; } public ApplicationId getApplicationId() { @@ -50,4 +53,7 @@ public class ResourceUsage { return plan; } + public NodeResources.Architecture getArchitecture() { + return architecture; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java index 6050f238da7..cb2b76d845c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMailer.java @@ -12,9 +12,25 @@ import java.util.Map; public class MockMailer implements Mailer { public final Map<String, List<Mail>> mails = new HashMap<>(); + public final boolean blackhole; + + public MockMailer() { + this(false); + } + + MockMailer(boolean blackhole) { + this.blackhole = blackhole; + } + + public static MockMailer blackhole() { + return new MockMailer(true); + } @Override public void send(Mail mail) { + if (blackhole) { + return; + } for (String recipient : mail.recipients()) { mails.putIfAbsent(recipient, new ArrayList<>()); mails.get(recipient).add(mail); @@ -33,7 +49,10 @@ public class MockMailer implements Mailer { /** Returns the list of mails sent to the given recipient. Modifications affect the set of mails stored in this. */ public List<Mail> inbox(String recipient) { - return mails.get(recipient); + return mails.getOrDefault(recipient, List.of()); } + public void reset() { + mails.clear(); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java index 003ecb32a32..ca094f98607 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java @@ -1,17 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import java.time.YearMonth; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,14 +29,6 @@ public class MockMeteringClient implements MeteringClient { } @Override - public MeteringData getMeteringData(TenantName tenantName, ApplicationName applicationName) { - return meteringData.orElseGet(() -> { - ResourceAllocation emptyAllocation = new ResourceAllocation(0, 0, 0); - return new MeteringData(emptyAllocation, emptyAllocation, emptyAllocation, Collections.emptyMap()); - }); - } - - @Override public List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth) { return new ArrayList<>(resources); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index cc667175316..98c64a2a11e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -228,8 +228,8 @@ enum PathGroup { hostedAccountant("/billing/v1/invoice/{*}", "/billing/v1/billing"), - /** Path used for listing endpoint certificate request info */ - endpointCertificateRequestInfo("/certificateRequests/"), + /** Path used for listing endpoint certificate request and re-requesting endpoint certificates */ + endpointCertificates("/endpointcertificates/"), /** Path used for secret store management */ secretStore(Matcher.tenant, "/application/v4/tenant/{tenant}/secret-store/{*}"), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index afecbf9d2e3..4d342e3b1ee 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -197,9 +197,9 @@ enum Policy { .on(PathGroup.hostedAccountant, PathGroup.accountant) .in(SystemName.PublicCd, SystemName.Public)), - /** Listing endpoint certificate request info */ - endpointCertificateRequestInfo(Privilege.grant(Action.read) - .on(PathGroup.endpointCertificateRequestInfo) + /** Listing endpoint certificates and re-requesting certificates */ + endpointCertificateApi(Privilege.grant(Action.all()) + .on(PathGroup.endpointCertificates) .in(SystemName.all())), /** Secret store operations */ diff --git a/controller-server/pom.xml b/controller-server/pom.xml index fa4a0dc06d6..ca8951124ef 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -138,7 +138,7 @@ <dependency> <groupId>commons-fileupload</groupId> <artifactId>commons-fileupload</artifactId> - <version>1.3.3</version> + <version>1.4</version> </dependency> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 6afeab9b4e6..c35e8c5a7ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.notification.NotificationsDb; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; import com.yahoo.vespa.hosted.controller.security.AccessControl; @@ -88,6 +89,7 @@ public class Controller extends AbstractComponent { private final CuratorArchiveBucketDb archiveBucketDb; private final NotificationsDb notificationsDb; private final SupportAccessControl supportAccessControl; + private final Notifier notifier; /** * Creates a controller @@ -126,6 +128,7 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); + notifier = new Notifier(curator, serviceRegistry.mailer()); notificationsDb = new NotificationsDb(this); supportAccessControl = new SupportAccessControl(this); @@ -330,4 +333,7 @@ public class Controller extends AbstractComponent { return supportAccessControl; } + public Notifier notifier() { + return notifier; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 59877fce634..384a5d0f1ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -171,7 +171,15 @@ public class TenantController { throw new IllegalArgumentException("Could not delete tenant '" + tenant.value() + "': This tenant has active applications"); - credentials.ifPresent(creds -> accessControl.deleteTenant(tenant, creds)); + if (oldTenant.type() == Tenant.Type.athenz) { + credentials.ifPresent(creds -> accessControl.deleteTenant(tenant, creds)); + } else if (oldTenant.type() == Tenant.Type.cloud) { + accessControl.deleteTenant(tenant, null); + } else { + throw new IllegalArgumentException("Could not delete tenant '" + tenant.value() + + ": This tenant is of unhandled type " + oldTenant.type()); + } + controller.notificationsDb().removeNotifications(NotificationSource.from(tenant)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 5c7ae5041fe..258884a4d11 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.controller.application.pkg; import com.google.common.hash.Funnel; import com.google.common.hash.Hashing; import com.yahoo.component.Version; +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; +import com.yahoo.compress.ArchiveStreamReader.Options; import com.yahoo.config.application.FileSystemWrapper; import com.yahoo.config.application.FileSystemWrapper.FileWrapper; import com.yahoo.config.application.XmlPreProcessor; @@ -24,6 +27,7 @@ import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,6 +44,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedMap; +import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.function.Function; import java.util.function.Predicate; @@ -108,7 +113,7 @@ public class ApplicationPackage { this.trustedCertificates = files.get(trustedCertificatesFile).map(bytes -> X509CertificateUtils.certificateListFromPem(new String(bytes, UTF_8))).orElse(List.of()); - this.bundleHash = calculateBundleHash(); + this.bundleHash = calculateBundleHash(zippedContent); preProcessAndPopulateCache(); } @@ -120,7 +125,7 @@ public class ApplicationPackage { byte[] certificatesBytes = X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8); ByteArrayOutputStream modified = new ByteArrayOutputStream(zippedContent.length + certificatesBytes.length); - ZipStreamReader.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); + ZipEntries.transferAndWrite(modified, new ByteArrayInputStream(zippedContent), trustedCertificatesFile, certificatesBytes); return new ApplicationPackage(modified.toByteArray()); } @@ -227,15 +232,23 @@ public class ApplicationPackage { } // Hashes all files and settings that require a deployment to be forwarded to configservers - private String calculateBundleHash() { + private String calculateBundleHash(byte[] zippedContent) { Predicate<String> entryMatcher = name -> ! name.endsWith(deploymentFile) && ! name.endsWith(buildMetaFile); - SortedMap<String, Long> entryCRCs = ZipStreamReader.getEntryCRCs(new ByteArrayInputStream(zippedContent), entryMatcher); - Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.entrySet().forEach(entry -> { - into.putBytes(entry.getKey().getBytes()); - into.putLong(entry.getValue()); + SortedMap<String, Long> crcByEntry = new TreeMap<>(); + Options options = Options.standard().pathPredicate(entryMatcher); + ArchiveFile file; + try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zippedContent), options)) { + OutputStream discard = OutputStream.nullOutputStream(); + while ((file = reader.readNextTo(discard)) != null) { + crcByEntry.put(file.path().toString(), file.crc32().orElse(-1)); + } + } + Funnel<SortedMap<String, Long>> funnel = (from, into) -> from.forEach((key, value) -> { + into.putBytes(key.getBytes()); + into.putLong(value); }); return Hashing.sha1().newHasher() - .putObject(entryCRCs, funnel) + .putObject(crcByEntry, funnel) .putInt(deploymentSpec.deployableHashCode()) .hash().toString(); } @@ -285,13 +298,13 @@ public class ApplicationPackage { } private Map<Path, Optional<byte[]>> read(Collection<String> names) { - var entries = new ZipStreamReader(new ByteArrayInputStream(zip), - name -> names.contains(withoutLegacyDir(name)), - maxSize, - true) - .entries().stream() - .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.zipEntry().getName())).normalize(), - ZipStreamReader.ZipEntryWithContent::content)); + var entries = ZipEntries.from(zip, + name -> names.contains(withoutLegacyDir(name)), + maxSize, + true) + .asList().stream() + .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.name())).normalize(), + ZipEntries.ZipEntryWithContent::content)); names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); return entries; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java index 97810b9de80..e18f6247cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageDiff.java @@ -15,7 +15,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader.ZipEntryWithContent; +import static com.yahoo.vespa.hosted.controller.application.pkg.ZipEntries.ZipEntryWithContent; /** * @author freva @@ -75,8 +75,8 @@ public class ApplicationPackageDiff { } private static Map<String, ZipEntryWithContent> readContents(ApplicationPackage app, int maxFileSizeToDiff) { - return new ZipStreamReader(new ByteArrayInputStream(app.zippedContent()), entry -> true, maxFileSizeToDiff, false).entries().stream() - .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), e -> e)); + return ZipEntries.from(app.zippedContent(), entry -> true, maxFileSizeToDiff, false).asList().stream() + .collect(Collectors.toMap(ZipEntryWithContent::name, e -> e)); } private static List<String> lines(byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java new file mode 100644 index 00000000000..a6cb7f23fc3 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -0,0 +1,99 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.compress.ArchiveStreamReader; +import com.yahoo.compress.ArchiveStreamReader.ArchiveFile; +import com.yahoo.compress.ArchiveStreamReader.Options; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** + * A list of entries read from a ZIP archive, and their contents. + * + * @author bratseth + */ +public class ZipEntries { + + private final List<ZipEntryWithContent> entries; + + private ZipEntries(List<ZipEntryWithContent> entries) { + this.entries = List.copyOf(Objects.requireNonNull(entries)); + } + + /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ + public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { + try (ZipOutputStream zipOut = new ZipOutputStream(out); + ZipInputStream zipIn = new ZipInputStream(in)) { + for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { + if (entry.getName().equals(name)) + continue; + + zipOut.putNextEntry(new ZipEntry(entry.getName())); + zipIn.transferTo(zipOut); + zipOut.closeEntry(); + } + zipOut.putNextEntry(new ZipEntry(name)); + zipOut.write(content); + zipOut.closeEntry(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** Read ZIP entries from inputStream */ + public static ZipEntries from(byte[] zip, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { + Options options = Options.standard() + .pathPredicate(entryNameMatcher) + .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB + .maxEntrySize(maxEntrySizeInBytes) + .maxEntries(1024) + .truncateEntry(!throwIfEntryExceedsMaxSize); + List<ZipEntryWithContent> entries = new ArrayList<>(); + try (ArchiveStreamReader reader = ArchiveStreamReader.ofZip(new ByteArrayInputStream(zip), options)) { + ArchiveFile file; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((file = reader.readNextTo(baos)) != null) { + entries.add(new ZipEntryWithContent(file.path().toString(), + Optional.of(baos.toByteArray()).filter(b -> b.length > 0), + file.size())); + baos.reset(); + } + } + return new ZipEntries(entries); + } + + public List<ZipEntryWithContent> asList() { return entries; } + + public static class ZipEntryWithContent { + + private final String name; + private final Optional<byte[]> content; + private final long size; + + public ZipEntryWithContent(String name, Optional<byte[]> content, long size) { + this.name = name; + this.content = content; + this.size = size; + } + + public String name() { return name; } + public byte[] contentOrThrow() { return content.orElseThrow(); } + public Optional<byte[]> content() { return content; } + public long size() { return size; } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java deleted file mode 100644 index 174ac4cb8b0..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReader.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -/** - * @author bratseth - */ -public class ZipStreamReader { - - private final List<ZipEntryWithContent> entries = new ArrayList<>(); - private final int maxEntrySizeInBytes; - - public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { - this.maxEntrySizeInBytes = maxEntrySizeInBytes; - try (ZipInputStream zipInput = new ZipInputStream(input)) { - ZipEntry zipEntry; - - while (null != (zipEntry = zipInput.getNextEntry())) { - if (!entryNameMatcher.test(requireName(zipEntry.getName()))) continue; - entries.add(readContent(zipEntry, zipInput, throwIfEntryExceedsMaxSize)); - } - } catch (IOException e) { - throw new UncheckedIOException("IO error reading zip content", e); - } - } - - /** Copies the zipped content from in to out, adding/overwriting an entry with the given name and content. */ - public static void transferAndWrite(OutputStream out, InputStream in, String name, byte[] content) { - try (ZipOutputStream zipOut = new ZipOutputStream(out); - ZipInputStream zipIn = new ZipInputStream(in)) { - for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if (entry.getName().equals(name)) - continue; - - zipOut.putNextEntry(new ZipEntry(entry.getName())); - zipIn.transferTo(zipOut); - zipOut.closeEntry(); - } - zipOut.putNextEntry(new ZipEntry(name)); - zipOut.write(content); - zipOut.closeEntry(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static SortedMap<String, Long> getEntryCRCs(InputStream in, Predicate<String> entryNameMatcher) { - SortedMap<String, Long> entryCRCs = new TreeMap<>(); - byte[] buffer = new byte[2048]; - try (ZipInputStream zipIn = new ZipInputStream(in)) { - for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) { - if ( ! entryNameMatcher.test(entry.getName())) - continue; - // CRC is not set until entry is read - while ( -1 != zipIn.read(buffer)){} - entryCRCs.put(entry.getName(), entry.getCrc()); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return entryCRCs; - } - - private ZipEntryWithContent readContent(ZipEntry zipEntry, ZipInputStream zipInput, boolean throwIfEntryExceedsMaxSize) { - try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) { - byte[] buffer = new byte[2048]; - int read; - long size = 0; - while ( -1 != (read = zipInput.read(buffer))) { - size += read; - if (size > maxEntrySizeInBytes) { - if (throwIfEntryExceedsMaxSize) throw new IllegalArgumentException( - "Entry in zip content exceeded size limit of " + maxEntrySizeInBytes + " bytes"); - } else bis.write(buffer, 0, read); - } - - boolean hasContent = size <= maxEntrySizeInBytes; - return new ZipEntryWithContent(zipEntry, - Optional.of(bis).filter(__ -> hasContent).map(ByteArrayOutputStream::toByteArray), - size); - } catch (IOException e) { - throw new UncheckedIOException("Failed reading from zipped content", e); - } - } - - public List<ZipEntryWithContent> entries() { return Collections.unmodifiableList(entries); } - - private static String requireName(String name) { - if (List.of(name.split("/")).contains("..") || - !trimTrailingSlash(name).equals(Path.of(name).normalize().toString())) { - throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); - } - return name; - } - - private static String trimTrailingSlash(String name) { - if (name.endsWith("/")) return name.substring(0, name.length() - 1); - return name; - } - - public static class ZipEntryWithContent { - - private final ZipEntry zipEntry; - private final Optional<byte[]> content; - private final long size; - - public ZipEntryWithContent(ZipEntry zipEntry, Optional<byte[]> content, long size) { - this.zipEntry = zipEntry; - this.content = content; - this.size = size; - } - - public ZipEntry zipEntry() { return zipEntry; } - public byte[] contentOrThrow() { return content.orElseThrow(); } - public Optional<byte[]> content() { return content; } - public long size() { return size; } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index 5e19b014083..996b53cc6f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -26,7 +26,7 @@ import java.util.stream.Collectors; /** * Looks up stored endpoint certificate metadata, provisions new certificates if none is found, - * re-provisions if zone is not covered, and uses refreshed certificates if a newer version is available. + * and re-provisions the certificate if the deploying-to zone is not covered. * * See also {@link com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer}, which handles * refreshes, deletions and triggers deployments. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesHandler.java new file mode 100644 index 00000000000..dc59f513509 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesHandler.java @@ -0,0 +1,77 @@ +package com.yahoo.vespa.hosted.controller.certificate; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.RestApiException; +import com.yahoo.restapi.StringResponse; +import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequestMetadata; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateMetadataSerializer; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; + +/** + * List all certificate requests for a system, with their requested DNS names. + * Used for debugging, and verifying basic functionality of Cameo client in CD. + * + * @author andreer + */ + +public class EndpointCertificatesHandler extends ThreadedHttpRequestHandler { + + private final EndpointCertificateProvider endpointCertificateProvider; + private final CuratorDb curator; + + public EndpointCertificatesHandler(Executor executor, ServiceRegistry serviceRegistry, CuratorDb curator) { + super(executor); + this.endpointCertificateProvider = serviceRegistry.endpointCertificateProvider(); + this.curator = curator; + } + + public HttpResponse handle(HttpRequest request) { + if (request.getMethod().equals(GET)) return listEndpointCertificates(); + if (request.getMethod().equals(POST)) return reRequestEndpointCertificateFor(request.getProperty("application")); + throw new RestApiException.MethodNotAllowed(request); + } + + public HttpResponse listEndpointCertificates() { + List<EndpointCertificateRequestMetadata> endpointCertificateMetadata = endpointCertificateProvider.listCertificates(); + + String requestsWithNames = endpointCertificateMetadata.stream() + .map(metadata -> metadata.requestId() + " : " + + String.join(", ", metadata.dnsNames().stream() + .map(dnsNameStatus -> dnsNameStatus.dnsName) + .collect(Collectors.joining(", ")))) + .collect(Collectors.joining("\n")); + + return new StringResponse(requestsWithNames); + } + + public StringResponse reRequestEndpointCertificateFor(String instanceId) { + ApplicationId applicationId = ApplicationId.fromFullString(instanceId); + + try (var lock = curator.lock(TenantAndApplicationId.from(applicationId))) { + EndpointCertificateMetadata endpointCertificateMetadata = curator.readEndpointCertificateMetadata(applicationId) + .orElseThrow(() -> new RestApiException.NotFound("No certificate found for application " + applicationId.serializedForm())); + + EndpointCertificateMetadata reRequestedMetadata = endpointCertificateProvider.requestCaSignedCertificate( + applicationId, endpointCertificateMetadata.requestedDnsSans(), Optional.of(endpointCertificateMetadata)); + + curator.writeEndpointCertificateMetadata(applicationId, reRequestedMetadata); + + return new StringResponse(EndpointCertificateMetadataSerializer.toSlime(reRequestedMetadata).toString()); + } + } +}
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index c9310349b9b..7ede040773e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -7,17 +7,24 @@ import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.time.Instant; import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; /** * Expires unused tenants from Vespa Cloud. * + * TODO: Should support sending notifications some time before the various expiry events happen. + * * @author ogronnesby */ public class CloudTrialExpirer extends ControllerMaintainer { @@ -33,39 +40,56 @@ public class CloudTrialExpirer extends ControllerMaintainer { @Override protected double maintain() { - var expiredTenants = controller().tenants().asList().stream() - .filter(this::tenantIsCloudTenant) // only valid for cloud tenants - .filter(this::tenantHasTrialPlan) // only valid to expire actual trial tenants - .filter(this::tenantIsNotExemptFromExpiry) // feature flag might exempt tenant from expiry - .filter(this::tenantReadersNotLoggedIn) // no user logged in last 14 days - .filter(this::tenantHasNoDeployments) // no running deployments active - .collect(Collectors.toList()); + if (controller().system().equals(SystemName.PublicCd)) { + tombstoneNonePlanTenants(); + } + moveInactiveTenantsToNonePlan(); + return 1.0; + } - if (! expiredTenants.isEmpty()) { - var expiredTenantNames = expiredTenants.stream() - .map(Tenant::name) - .map(TenantName::value) - .collect(Collectors.joining(", ")); + private void moveInactiveTenantsToNonePlan() { + var predicate = tenantReadersNotLoggedIn(loginExpiry) + .and(this::tenantHasTrialPlan) + .and(this::tenantHasNoDeployments); - log.info("Moving expired tenants to 'none' plan: " + expiredTenantNames); - } + forTenant("'none' plan", predicate, this::setPlanNone); + } + + private void tombstoneNonePlanTenants() { + // tombstone tenants that are inactive 14 days after being set as 'none' + var expiry = loginExpiry.plus(loginExpiry); + var predicate = tenantReadersNotLoggedIn(expiry).and(this::tenantHasNonePlan); + forTenant("tombstoned", predicate, this::tombstoneTenants); + } - expireTenants(expiredTenants); + private void forTenant(String name, Predicate<Tenant> p, Consumer<List<Tenant>> c) { + var predicate = ((Predicate<Tenant>) this::tenantIsCloudTenant) + .and(this::tenantIsNotExemptFromExpiry); + + var tenants = controller().tenants().asList().stream() + .filter(predicate.and(p)) + .collect(Collectors.toList()); - return 1; + if (! tenants.isEmpty()) { + var tenantNames = tenants.stream().map(Tenant::name).map(TenantName::value).collect(Collectors.joining(", ")); + log.info("Setting tenants as " + name + ": " + tenantNames); + } + + c.accept(tenants); } private boolean tenantIsCloudTenant(Tenant tenant) { return tenant.type() == Tenant.Type.cloud; } - private boolean tenantReadersNotLoggedIn(Tenant tenant) { - return tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.user) - .map(instant -> { - var sinceLastLogin = Duration.between(instant, controller().clock().instant()); - return sinceLastLogin.compareTo(loginExpiry) > 0; - }) - .orElse(false); + private Predicate<Tenant> tenantReadersNotLoggedIn(Duration duration) { + // returns true if no user has logged in to the tenant after (now - duration) + return (Tenant tenant) -> { + var timeLimit = controller().clock().instant().minus(duration); + return tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.user) + .map(instant -> instant.isBefore(timeLimit)) + .orElse(false); + }; } private boolean tenantHasTrialPlan(Tenant tenant) { @@ -73,6 +97,11 @@ public class CloudTrialExpirer extends ControllerMaintainer { return "trial".equals(planId.value()); } + private boolean tenantHasNonePlan(Tenant tenant) { + var planId = controller().serviceRegistry().billingController().getPlan(tenant.name()); + return "none".equals(planId.value()); + } + private boolean tenantIsNotExemptFromExpiry(Tenant tenant) { return ! extendedTrialTenants.value().contains(tenant.name().value()); } @@ -84,9 +113,15 @@ public class CloudTrialExpirer extends ControllerMaintainer { .sum() == 0; } - private void expireTenants(List<Tenant> tenants) { + private void setPlanNone(List<Tenant> tenants) { tenants.forEach(tenant -> { controller().serviceRegistry().billingController().setPlan(tenant.name(), PlanId.from("none"), false); }); } + + private void tombstoneTenants(List<Tenant> tenants) { + tenants.forEach(tenant -> { + controller().tenants().delete(tenant.name(), Optional.empty(), false); + }); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index b996901c5d0..15f8d6380c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -34,7 +34,7 @@ import java.util.stream.Collectors; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. * <p> - * See also EndpointCertificateManager, which provisions, reprovisions and validates certificates on deploy + * See also class EndpointCertificates, which provisions, reprovisions and validates certificates on deploy * * @author andreer */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index acaf35133d7..d4905f7e20a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -33,7 +33,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; +import java.util.function.Function; import java.util.logging.Level; +import java.util.stream.Collector; import java.util.stream.Collectors; /** @@ -150,14 +152,14 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) .filter(this::isClusterTypeMeterable) + // Grouping by ApplicationId -> Architecture -> ResourceSnapshot .collect(Collectors.groupingBy(node -> - node.owner().get(), - Collectors.collectingAndThen(Collectors.toList(), - nodeList -> ResourceSnapshot.from( - nodeList, - clock.instant(), - zoneId)) - )).values(); + node.owner().get(), + groupSnapshotsByArchitecture(zoneId))) + .values() + .stream() + .flatMap(list -> list.values().stream()) + .collect(Collectors.toList()); } private boolean unlessNodeOwnerIsSystemApplication(Node node) { @@ -182,7 +184,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { public static double cost(ClusterResources clusterResources, SystemName systemName) { NodeResources nr = clusterResources.nodeResources(); - return cost(new ResourceAllocation(nr.vcpu(), nr.memoryGb(), nr.diskGb()).multiply(clusterResources.nodes()), systemName); + return cost(new ResourceAllocation(nr.vcpu(), nr.memoryGb(), nr.diskGb(), nr.architecture()).multiply(clusterResources.nodes()), systemName); } private static double cost(ResourceAllocation allocation, SystemName systemName) { @@ -201,7 +203,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { metric.createContext(Collections.emptyMap())); resourceSnapshots.forEach(snapshot -> { - var context = getMetricContext(snapshot.getApplicationId(), snapshot.getZoneId()); + var context = getMetricContext(snapshot); metric.set("metering.vcpu", snapshot.getCpuCores(), context); metric.set("metering.memoryGB", snapshot.getMemoryGb(), context); metric.set("metering.diskGB", snapshot.getDiskGb(), context); @@ -222,4 +224,29 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { "zoneId", zoneId.value() )); } + + private Metric.Context getMetricContext(ResourceSnapshot snapshot) { + return metric.createContext(Map.of( + "tenant", snapshot.getApplicationId().tenant().value(), + "applicationId", snapshot.getApplicationId().toFullString(), + "zoneId", snapshot.getZoneId().value(), + "architecture", snapshot.getArchitecture() + )); + } + + private Collector<Node, ?, Map<NodeResources.Architecture, ResourceSnapshot>> groupSnapshotsByArchitecture(ZoneId zoneId) { + return Collectors.collectingAndThen( + Collectors.groupingBy(node -> node.resources().architecture()), + convertNodeListToResourceSnapshot(zoneId) + ); + } + + private Function<Map<NodeResources.Architecture, List<Node>>, Map<NodeResources.Architecture, ResourceSnapshot>> convertNodeListToResourceSnapshot(ZoneId zoneId) { + return nodeMap -> nodeMap.entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> ResourceSnapshot.from(entry.getValue(), clock.instant(), zoneId)) + ); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java index 089767dc586..b36b2b9cad8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java @@ -57,7 +57,7 @@ public class CostCalculator { Property property = propertyByTenantName.get(node.owner().get().tenant()); if (property == null) continue; var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO); - var nodeAllocation = new ResourceAllocation(node.resources().vcpu(), node.resources().memoryGb(), node.resources().diskGb()); + var nodeAllocation = new ResourceAllocation(node.resources().vcpu(), node.resources().memoryGb(), node.resources().diskGb(), node.resources().architecture()); allocationByProperty.put(property, allocation.plus(nodeAllocation)); totalAllocation = totalAllocation.plus(nodeAllocation); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index c0bd1ac03ff..aa62028749b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -32,14 +33,16 @@ public class NotificationsDb { private final Clock clock; private final CuratorDb curatorDb; + private final Notifier notifier; public NotificationsDb(Controller controller) { - this(controller.clock(), controller.curator()); + this(controller.clock(), controller.curator(), controller.notifier()); } - NotificationsDb(Clock clock, CuratorDb curatorDb) { + NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) { this.clock = clock; this.curatorDb = curatorDb; + this.notifier = notifier; } public List<TenantName> listTenantsWithNotifications() { @@ -61,13 +64,20 @@ public class NotificationsDb { * already exists, it'll be replaced by this one instead */ public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + Optional<Notification> changed = Optional.empty(); try (Lock lock = curatorDb.lockNotifications(source.tenant())) { - List<Notification> notifications = curatorDb.readNotifications(source.tenant()).stream() + var existingNotifications = curatorDb.readNotifications(source.tenant()); + List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) .collect(Collectors.toCollection(ArrayList::new)); - notifications.add(new Notification(clock.instant(), type, level, source, messages)); + var notification = new Notification(clock.instant(), type, level, source, messages); + if (!notificationExists(notification, existingNotifications, false)) { + changed = Optional.of(notification); + } + notifications.add(notification); curatorDb.writeNotifications(source.tenant(), notifications); } + changed.ifPresent(notifier::dispatch); } /** Remove the notification with the given source and type */ @@ -109,6 +119,7 @@ public class NotificationsDb { */ public void setDeploymentMetricsNotifications(DeploymentId deploymentId, List<ClusterMetrics> clusterMetrics) { Instant now = clock.instant(); + List<Notification> changed = List.of(); List<Notification> newNotifications = clusterMetrics.stream() .flatMap(metric -> { NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(metric.getClusterId())); @@ -130,10 +141,20 @@ public class NotificationsDb { // ... and add the new notifications for this deployment newNotifications.stream()) .collect(Collectors.toUnmodifiableList()); - - if (!initial.equals(updated)) + if (!initial.equals(updated)) { curatorDb.writeNotifications(deploymentSource.tenant(), updated); + } + changed = newNotifications.stream().filter(n -> !notificationExists(n, initial, true)).collect(Collectors.toList()); } + notifier.dispatch(changed, deploymentSource); + } + + private boolean notificationExists(Notification notification, List<Notification> existing, boolean mindHigherLevel) { + // Be conservative for now, only dispatch notifications if they are from new source or with new type. + // the message content and level is ignored for now + return existing.stream().anyMatch(e -> + notification.source().contains(e.source()) && notification.type().equals(e.type()) && + (!mindHigherLevel || notification.level().ordinal() <= e.level().ordinal())); } private static Optional<Notification> createFeedBlockNotification(NotificationSource source, Instant at, ClusterMetrics metric) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java new file mode 100644 index 00000000000..e186541c85c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notify/Notifier.java @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.notify; + +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; +import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Notifier is responsible for dispatching user notifications to their chosen Contact points. + * + * @author enygaard + */ +public class Notifier { + private final CuratorDb curatorDb; + private final Mailer mailer; + + private static final Logger log = Logger.getLogger(Notifier.class.getName()); + + public Notifier(CuratorDb curatorDb, Mailer mailer) { + this.curatorDb = Objects.requireNonNull(curatorDb); + this.mailer = Objects.requireNonNull(mailer); + } + + public void dispatch(List<Notification> notifications, NotificationSource source) { + if (notifications.isEmpty()) { + return; + } + var tenant = curatorDb.readTenant(source.tenant()); + tenant.stream().forEach(t -> { + if (t instanceof CloudTenant) { + var ct = (CloudTenant) t; + ct.info().contacts().all().stream() + .filter(c -> c.audiences().contains(TenantContacts.Audience.NOTIFICATIONS)) + .collect(Collectors.groupingBy(TenantContacts.Contact::type, Collectors.toList())) + .entrySet() + .forEach(e -> notifications.forEach(n -> dispatch(n, e.getKey(), e.getValue()))); + } + }); + } + + public void dispatch(Notification notification) { + dispatch(List.of(notification), notification.source()); + } + + private void dispatch(Notification notification, TenantContacts.Type type, Collection<? extends TenantContacts.Contact> contacts) { + switch (type) { + case EMAIL: + dispatch(notification, contacts.stream().map(c -> (TenantContacts.EmailContact) c).collect(Collectors.toList())); + break; + default: + throw new IllegalArgumentException("Unknown TenantContacts type " + type.name()); + } + } + + private void dispatch(Notification notification, Collection<TenantContacts.EmailContact> contacts) { + try { + mailer.send(mailOf(notification, contacts.stream().map(c -> c.email()).collect(Collectors.toList()))); + } catch (MailerException e) { + log.log(Level.SEVERE, "Failed sending email", e); + } + } + + private Mail mailOf(Notification n, Collection<String> recipients) { + var subject = Text.format("[%s] Vespa Notification for %s", n.level().toString().toUpperCase(), n.type().name()); + var body = new StringBuilder(); + body.append("Source: ").append(n.source().toString()).append("\n") + .append("\n") + .append(String.join("\n", n.messages())); + return new Mail(recipients, subject.toString(), body.toString()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index 2a1b8a19475..7e1c9c8884f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.HttpURL; +import com.yahoo.restapi.HttpURL.Path; +import com.yahoo.restapi.HttpURL.Query; import com.yahoo.text.Text; import java.io.InputStream; import java.net.URI; @@ -22,28 +25,27 @@ import static com.yahoo.jdisc.http.HttpRequest.Method; public class ProxyRequest { private final Method method; - private final URI requestUri; + private final HttpURL requestUri; private final Map<String, List<String>> headers; private final InputStream requestData; private final List<URI> targets; - private final String targetPath; + private final Path targetPath; - ProxyRequest(Method method, URI url, Map<String, List<String>> headers, InputStream body, List<URI> targets, - String path) { - Objects.requireNonNull(url); - if (!url.getPath().endsWith(path)) { - throw new IllegalArgumentException(Text.format("Request path '%s' does not end with proxy path '%s'", url.getPath(), path)); + ProxyRequest(Method method, URI uri, Map<String, List<String>> headers, InputStream body, List<URI> targets, Path path) { + this.requestUri = HttpURL.from(uri); + if ( requestUri.path().segments().size() < path.segments().size() + || ! requestUri.path().tail(path.segments().size()).equals(path)) { + throw new IllegalArgumentException(Text.format("Request %s does not end with proxy %s", requestUri.path(), path)); } if (targets.isEmpty()) { throw new IllegalArgumentException("targets must be non-empty"); } this.method = Objects.requireNonNull(method); - this.requestUri = Objects.requireNonNull(url); this.headers = Objects.requireNonNull(headers); this.requestData = body; this.targets = List.copyOf(targets); - this.targetPath = path.startsWith("/") ? path : "/" + path; + this.targetPath = path; } @@ -64,24 +66,12 @@ public class ProxyRequest { } public URI createConfigServerRequestUri(URI baseURI) { - try { - return new URI(baseURI.getScheme(), baseURI.getUserInfo(), baseURI.getHost(), - baseURI.getPort(), targetPath, requestUri.getQuery(), requestUri.getFragment()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + return HttpURL.from(baseURI).withPath(targetPath).withQuery(requestUri.query()).asURI(); } public URI getControllerPrefixUri() { - String prefixPath = targetPath.equals("/") && !requestUri.getPath().endsWith("/") ? - requestUri.getPath() + targetPath : - requestUri.getPath().substring(0, requestUri.getPath().length() - targetPath.length() + 1); - try { - return new URI(requestUri.getScheme(), requestUri.getUserInfo(), requestUri.getHost(), - requestUri.getPort(), prefixPath, null, null); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + Path prefixPath = requestUri.path().cut(targetPath.segments().size()).withTrailingSlash(); + return requestUri.withPath(prefixPath).withQuery(Query.empty()).asURI(); } @Override @@ -90,7 +80,7 @@ public class ProxyRequest { } /** Create a proxy request that repeatedly tries a single target */ - public static ProxyRequest tryOne(URI target, String path, HttpRequest request) { + public static ProxyRequest tryOne(URI target, Path path, HttpRequest request) { return new ProxyRequest(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), request.getData(), List.of(target), path); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 0b629a577d0..9ac30898f8b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL; +import com.yahoo.restapi.HttpURL.Path; import org.apache.http.client.utils.URIBuilder; import java.io.IOException; @@ -29,19 +31,8 @@ public class ProxyResponse extends HttpResponse { super(statusResponse); this.contentType = contentType; - final String configServerPrefix; - final String controllerRequestPrefix; - try { - configServerPrefix = new URIBuilder() - .setScheme(configServer.getScheme()) - .setHost(configServer.getHost()) - .setPort(configServer.getPort()) - .setPath("/") - .build().toString(); - controllerRequestPrefix = controllerRequest.getControllerPrefixUri().toString(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + String configServerPrefix = HttpURL.from(configServer).withPath(Path.empty()).asURI().toString(); + String controllerRequestPrefix = controllerRequest.getControllerPrefixUri().toString(); bodyResponseRewritten = bodyResponse.replace(configServerPrefix, controllerRequestPrefix); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d9a38a5b578..0f5322af176 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; +import ai.vespa.validation.Validation; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -25,8 +26,10 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.net.DomainName; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.HttpURL; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; @@ -155,6 +158,7 @@ import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; +import static ai.vespa.validation.Validation.requireAtLeast; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.yolean.Exceptions.uncheck; @@ -255,7 +259,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/diff/{number}")) return applicationPackageDiff(path.get("tenant"), path.get("application"), path.get("number")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/metering")) return metering(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); @@ -270,7 +273,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); @@ -284,7 +288,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/state/v1/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{host}/status/{*}")) return status(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.get("host"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); @@ -1762,68 +1767,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse metering(String tenant, String application, HttpRequest request) { - - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - MeteringData meteringData = controller.serviceRegistry() - .meteringService() - .getMeteringData(TenantName.from(tenant), ApplicationName.from(application)); - - ResourceAllocation currentSnapshot = meteringData.getCurrentSnapshot(); - Cursor currentRate = root.setObject("currentrate"); - currentRate.setDouble("cpu", currentSnapshot.getCpuCores()); - currentRate.setDouble("mem", currentSnapshot.getMemoryGb()); - currentRate.setDouble("disk", currentSnapshot.getDiskGb()); - - ResourceAllocation thisMonth = meteringData.getThisMonth(); - Cursor thismonth = root.setObject("thismonth"); - thismonth.setDouble("cpu", thisMonth.getCpuCores()); - thismonth.setDouble("mem", thisMonth.getMemoryGb()); - thismonth.setDouble("disk", thisMonth.getDiskGb()); - - ResourceAllocation lastMonth = meteringData.getLastMonth(); - Cursor lastmonth = root.setObject("lastmonth"); - lastmonth.setDouble("cpu", lastMonth.getCpuCores()); - lastmonth.setDouble("mem", lastMonth.getMemoryGb()); - lastmonth.setDouble("disk", lastMonth.getDiskGb()); - - - Map<ApplicationId, List<ResourceSnapshot>> history = meteringData.getSnapshotHistory(); - Cursor details = root.setObject("details"); - - Cursor detailsCpu = details.setObject("cpu"); - Cursor detailsMem = details.setObject("mem"); - Cursor detailsDisk = details.setObject("disk"); - - history.forEach((applicationId, resources) -> { - String instanceName = applicationId.instance().value(); - Cursor detailsCpuApp = detailsCpu.setObject(instanceName); - Cursor detailsMemApp = detailsMem.setObject(instanceName); - Cursor detailsDiskApp = detailsDisk.setObject(instanceName); - Cursor detailsCpuData = detailsCpuApp.setArray("data"); - Cursor detailsMemData = detailsMemApp.setArray("data"); - Cursor detailsDiskData = detailsDiskApp.setArray("data"); - - resources.forEach(resourceSnapshot -> { - Cursor cpu = detailsCpuData.addObject(); - cpu.setLong("unixms", resourceSnapshot.getTimestamp().toEpochMilli()); - cpu.setDouble("value", resourceSnapshot.getCpuCores()); - - Cursor mem = detailsMemData.addObject(); - mem.setLong("unixms", resourceSnapshot.getTimestamp().toEpochMilli()); - mem.setDouble("value", resourceSnapshot.getMemoryGb()); - - Cursor disk = detailsDiskData.addObject(); - disk.setLong("unixms", resourceSnapshot.getTimestamp().toEpochMilli()); - disk.setDouble("value", resourceSnapshot.getDiskGb()); - }); - }); - - return new SlimeJsonResponse(slime); - } - private HttpResponse deploying(String tenantName, String applicationName, String instanceName, HttpRequest request) { Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); Slime slime = new Slime(); @@ -1857,40 +1800,29 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return response; } - private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { + private HttpResponse status(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String host, HttpURL.Path restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); + String result = controller.serviceRegistry().configServer().getServiceStatusPage(deploymentId, + serviceName, + DomainName.of(host), + restPath); // TODO: add query + return new HtmlResponse(result); + } - if (restPath.contains("/status/")) { - String[] parts = restPath.split("/status/", 2); - String result = controller.serviceRegistry().configServer().getServiceStatusPage(deploymentId, serviceName, parts[0], parts[1]); - return new HtmlResponse(result); - } - - String normalizedRestPath = URI.create(restPath).normalize().toString(); - // Only state/v1 is allowed - if (! normalizedRestPath.startsWith("state/v1/")) { - return ErrorResponse.forbidden("Access denied"); - } - + private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, HttpURL.Path restPath, HttpRequest request) { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), deploymentId.applicationId(), List.of(controller.zoneRegistry().getConfigServerVipUri(deploymentId.zoneId())), request.getUri()); - response.setResponse(result, serviceName, restPath); + response.setResponse(result, serviceName, HttpURL.Path.parse("/state/v1").append(restPath)); return response; } - private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, String restPath, HttpRequest request) { + private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, HttpURL.Path restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - - String normalizedRestPath = URI.create("content/" + restPath).normalize().toString(); - // Only content/ is allowed - if ( ! normalizedRestPath.startsWith("content/")) { - return ErrorResponse.forbidden("Access denied"); - } - - return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, "/" + restPath, request.getUri()); + return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, restPath, request.getUri()); } private HttpResponse updateTenant(String tenantName, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java index 8d03ca74500..a9e24943c0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java @@ -5,6 +5,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import org.apache.commons.fileupload.MultipartStream; import org.apache.commons.fileupload.ParameterParser; +import org.apache.commons.fileupload.util.Streams; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -21,6 +22,16 @@ import java.util.Map; */ public class MultipartParser { + private final long maxDataLength; + + public MultipartParser() { + this(2 * (long) Math.pow(1024, 3)); // 2 GB + } + + MultipartParser(long maxDataLength) { + this.maxDataLength = maxDataLength; + } + /** * Parses the given multi-part request and returns all the parts indexed by their name. * @@ -37,10 +48,13 @@ public class MultipartParser { */ public Map<String, byte[]> parse(String contentTypeHeader, InputStream data, URI uri) { try { + LimitedOutputStream output = new LimitedOutputStream(maxDataLength); ParameterParser parameterParser = new ParameterParser(); Map<String, String> contentType = parameterParser.parse(contentTypeHeader, ';'); - if (contentType.containsKey("application/zip")) - return Map.of(EnvironmentResource.APPLICATION_ZIP, data.readAllBytes()); + if (contentType.containsKey("application/zip")) { + Streams.copy(data, output, false); + return Map.of(EnvironmentResource.APPLICATION_ZIP, output.toByteArray()); + } if ( ! contentType.containsKey("multipart/form-data")) throw new IllegalArgumentException("Expected a multipart or application/zip message, but got Content-Type: " + contentTypeHeader); String boundary = contentType.get("boundary"); @@ -55,17 +69,17 @@ public class MultipartParser { if (contentDispositionContent == null) throw new IllegalArgumentException("Missing Content-Disposition header in a multipart body part"); Map<String, String> contentDisposition = parameterParser.parse(contentDispositionContent, ';'); - ByteArrayOutputStream output = new ByteArrayOutputStream(); multipartStream.readBodyData(output); parts.put(contentDisposition.get("name"), output.toByteArray()); + output.reset(); nextPart = multipartStream.readBoundary(); } return parts; } - catch(MultipartStream.MalformedStreamException e) { + catch (MultipartStream.MalformedStreamException e) { throw new IllegalArgumentException("Malformed multipart/form-data request", e); } - catch(IOException e) { + catch (IOException e) { throw new IllegalArgumentException("IO error reading multipart request " + uri, e); } } @@ -80,4 +94,34 @@ public class MultipartParser { return null; } + /** A {@link java.io.ByteArrayOutputStream} that limits the number of bytes written to it */ + private static class LimitedOutputStream extends ByteArrayOutputStream { + + private long remaining; + + /** Create a new OutputStream that can fit up to len bytes */ + private LimitedOutputStream(long len) { + this.remaining = len; + } + + @Override + public synchronized void write(int b) { + requireCapacity(1); + super.write(b); + remaining--; + } + + @Override + public synchronized void write(byte[] b, int off, int len) { + requireCapacity(len); + super.write(b, off, len); + remaining -= len; + } + + private void requireCapacity(int len) { + if (len > remaining) throw new IllegalArgumentException("Too many bytes to write"); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index 888c402b6ec..a21f93bdaea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL; +import com.yahoo.restapi.HttpURL.Path; +import com.yahoo.restapi.HttpURL.Query; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; @@ -34,11 +37,11 @@ class ServiceApiResponse extends HttpResponse { private final ApplicationId application; private final List<URI> configServerURIs; private final Slime slime; - private final UriBuilder requestUri; + private final HttpURL requestUri; // Only set for one of the setResponse calls private String serviceName = null; - private String restPath = null; + private Path restPath = null; public ServiceApiResponse(ZoneId zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { super(200); @@ -46,7 +49,7 @@ class ServiceApiResponse extends HttpResponse { this.application = application; this.configServerURIs = configServerURIs; this.slime = new Slime(); - this.requestUri = new UriBuilder(requestUri).withoutParameters(); + this.requestUri = HttpURL.from(requestUri).withQuery(Query.empty()); } public void setResponse(ApplicationView applicationView) { @@ -68,7 +71,7 @@ class ServiceApiResponse extends HttpResponse { } } - public void setResponse(Map<?,?> responseData, String serviceName, String restPath) { + public void setResponse(Map<?,?> responseData, String serviceName, Path restPath) { this.serviceName = serviceName; this.restPath = restPath; mapToSlime(responseData, slime.setObject()); @@ -138,7 +141,7 @@ class ServiceApiResponse extends HttpResponse { mapToSlime((Map)entry, array.addObject()); } - private String rewriteIfUrl(String urlOrAnyString, UriBuilder requestUri) { + private String rewriteIfUrl(String urlOrAnyString, HttpURL requestUri) { if (urlOrAnyString == null) return null; String hostPattern = "(" + @@ -163,19 +166,18 @@ class ServiceApiResponse extends HttpResponse { if (matcher.find()) { String proxiedPath = urlOrAnyString.substring(matcher.group().length()); - return requestUri.append(proxiedPath).toString(); + return requestUri.withPath(requestUri.path().append(Path.parse(proxiedPath))).asURI().toString(); } else { return urlOrAnyString; // not a service url } } - private UriBuilder generateLocalLinkPrefix(String identifier, String restPath) { - String proxiedPath = identifier + "/" + restPath; - - if (this.requestUri.toString().endsWith(proxiedPath)) { - return new UriBuilder(this.requestUri.toString().substring(0, this.requestUri.toString().length() - proxiedPath.length())); + private HttpURL generateLocalLinkPrefix(String identifier, Path restPath) { + Path proxiedPath = Path.parse(identifier).append(restPath); + if (requestUri.path().tail(proxiedPath.segments().size()).equals(proxiedPath)) { + return requestUri.withPath(requestUri.path().cut(proxiedPath.segments().size())); } else { - throw new IllegalStateException("Expected the resource path '" + this.requestUri + "' to end with '" + proxiedPath + "'"); + throw new IllegalStateException("Expected the resource " + requestUri.path() + " to end with " + proxiedPath); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 0e09825ec41..27a8cbeaf3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; @@ -21,8 +22,11 @@ import com.yahoo.yolean.Exceptions; import java.net.URI; import java.util.List; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.restapi.HttpURL.Path.parse; + /** * REST API for proxying operator APIs to config servers in a given zone. * @@ -32,7 +36,9 @@ import java.util.stream.Stream; public class ConfigServerApiHandler extends AuditLoggingRequestHandler { private static final URI CONTROLLER_URI = URI.create("https://localhost:4443/"); - private static final List<String> WHITELISTED_APIS = List.of("/flags/v1/", "/nodes/v2/", "/orchestrator/v1/"); + private static final List<HttpURL.Path> WHITELISTED_APIS = List.of(parse("/flags/v1/"), + parse("/nodes/v2/"), + parse("/orchestrator/v1/")); private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; @@ -84,17 +90,18 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { } ZoneId zoneId = ZoneId.from(path.get("environment"), path.get("region")); - if (! zoneRegistry.hasZone(zoneId) && ! controllerZone.equals(zoneId)) { + if ( ! zoneRegistry.hasZone(zoneId) && ! controllerZone.equals(zoneId)) { throw new IllegalArgumentException("No such zone: " + zoneId.value()); } - String cfgPath = "/" + path.getRest(); - if (WHITELISTED_APIS.stream().noneMatch(cfgPath::startsWith)) { - return ErrorResponse.forbidden("Cannot access '" + cfgPath + - "' through /configserver/v1, following APIs are permitted: " + String.join(", ", WHITELISTED_APIS)); + if (path.getRest().segments().size() < 2 || ! WHITELISTED_APIS.contains(path.getRest().head(2).withTrailingSlash())) { + return ErrorResponse.forbidden("Cannot access " + path.getRest() + + " through /configserver/v1, following APIs are permitted: " + WHITELISTED_APIS.stream() + .map(p -> "/" + String.join("/", p.segments()) + "/") + .collect(Collectors.joining(", "))); } - return proxy.handle(ProxyRequest.tryOne(getEndpoint(zoneId), cfgPath, request)); + return proxy.handle(ProxyRequest.tryOne(getEndpoint(zoneId), path.getRest(), request)); } private HttpResponse root(HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java index 850a9ef6107..33cd4948a7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java @@ -32,6 +32,7 @@ public class MeteringResponse extends SlimeJsonResponse { object.setDouble("cpu", snapshot.getCpuCores()); object.setDouble("memory", snapshot.getMemoryGb()); object.setDouble("disk", snapshot.getDiskGb()); + object.setString("architecture", snapshot.getArchitecture().name()); }); return slime; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 158cc6caede..1a4a42cb521 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -30,6 +30,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -132,7 +133,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { entry -> entry.getValue().instanceJobs().get(entry.getKey()))); Cursor productionArray = versionObject.setArray("productionApplications"); statistics.productionSuccesses().stream() - .collect(groupingBy(run -> run.id().application())) + .collect(groupingBy(run -> run.id().application(), TreeMap::new, toList())) .forEach((id, runs) -> { Cursor applicationObject = productionArray.addObject(); toSlime(applicationObject, id, request); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 766a51e3e8d..a7472ced09c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -4,12 +4,10 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.auth0.jwt.JWT; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; import com.yahoo.restapi.Path; @@ -90,6 +88,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } catch (Exception e) { logger.log(Level.INFO, () -> "Exception mapping Athenz principal to roles: " + Exceptions.toMessageString(e)); + return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access denied")); } return Optional.empty(); } @@ -104,7 +103,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { Optional<ApplicationName> application = Optional.ofNullable(path.get("application")).map(ApplicationName::from); final Optional<ZoneId> zone; - if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index a7416cfa0ed..40a402f209b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.HttpURL; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; @@ -107,7 +108,7 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private ProxyRequest proxyRequest(ZoneId zoneId, String path, HttpRequest request) { + private ProxyRequest proxyRequest(ZoneId zoneId, HttpURL.Path path, HttpRequest request) { return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index de70fc54d3b..39ee2a6ce44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -117,7 +117,6 @@ public class CloudAccessControl implements AccessControl { @Override public void deleteTenant(TenantName tenant, Credentials credentials) { - deleteBillingInfo(tenant, credentials); for (TenantRole role : Roles.tenantRoles(tenant)) userManagement.deleteRole(role); } @@ -134,13 +133,4 @@ public class CloudAccessControl implements AccessControl { userManagement.deleteRole(role); } - private void deleteBillingInfo(TenantName tenant, Credentials credentials) { - var users = Roles.tenantRoles(tenant) - .stream() - .flatMap(role -> userManagement.listUsers(role).stream()) - .collect(Collectors.toSet()); - var isPrivileged = allowedByPrivilegedRole((Auth0Credentials) credentials); - billingController.deleteBillingInfo(tenant, users, isPrivileged); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 709a7967a5e..4e155e937b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -154,9 +153,9 @@ public class ApplicationPackageTest { } private static Map<String, String> unzip(byte[] zip) { - return new ZipStreamReader(new ByteArrayInputStream(zip), __ -> true, 1 << 10, true) - .entries().stream() - .collect(Collectors.toMap(entry -> entry.zipEntry().getName(), + return ZipEntries.from(zip, __ -> true, 1 << 10, true) + .asList().stream() + .collect(Collectors.toMap(ZipEntries.ZipEntryWithContent::name, entry -> new String(entry.contentOrThrow(), UTF_8))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java new file mode 100644 index 00000000000..6908464640b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntriesTest.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application.pkg; + +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ZipEntriesTest { + + @Test + public void test_replacement() { + ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); + List<X509Certificate> certificates = IntStream.range(0, 3) + .mapToObj(i -> { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal subject = new X500Principal("CN=subject" + i); + return X509CertificateBuilder.fromKeypair(keyPair, + subject, + Instant.now(), + Instant.now().plusSeconds(1), + SignatureAlgorithm.SHA512_WITH_ECDSA, + BigInteger.valueOf(1)) + .build(); + }) + .collect(Collectors.toUnmodifiableList()); + + assertEquals(List.of(), applicationPackage.trustedCertificates()); + for (int i = 0; i < certificates.size(); i++) { + applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); + assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java deleted file mode 100644 index 33c18d123d2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipStreamReaderTest.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application.pkg; - -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import org.junit.Test; - -import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author mpolden - */ -public class ZipStreamReaderTest { - - @Test - public void test_size_limit() { - Map<String, String> entries = Map.of("foo.xml", "foobar"); - try { - new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 1, true); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} - - entries = Map.of("foo.xml", "foobar", - "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit - ); - ZipStreamReader reader = new ZipStreamReader(new ByteArrayInputStream(zip(entries)), "foo.xml"::equals, 10, true); - byte[] extracted = reader.entries().get(0).contentOrThrow(); - assertEquals("foobar", new String(extracted, StandardCharsets.UTF_8)); - } - - @Test - public void test_paths() { - Map<String, Boolean> tests = Map.of( - "../../services.xml", true, - "/../.././services.xml", true, - "./application/././services.xml", true, - "application//services.xml", true, - "artifacts/", false, // empty dir - "services..xml", false, - "application/services.xml", false, - "components/foo-bar-deploy.jar", false, - "services.xml", false - ); - tests.forEach((name, expectException) -> { - try { - new ZipStreamReader(new ByteArrayInputStream(zip(Map.of(name, "foo"))), name::equals, 1024, true); - assertFalse("Expected exception for '" + name + "'", expectException); - } catch (IllegalArgumentException ignored) { - assertTrue("Unexpected exception for '" + name + "'", expectException); - } - }); - } - - @Test - public void test_replacement() { - ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); - List<X509Certificate> certificates = IntStream.range(0, 3) - .mapToObj(i -> { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - X500Principal subject = new X500Principal("CN=subject" + i); - return X509CertificateBuilder.fromKeypair(keyPair, - subject, - Instant.now(), - Instant.now().plusSeconds(1), - SignatureAlgorithm.SHA512_WITH_ECDSA, - BigInteger.valueOf(1)) - .build(); - }) - .collect(Collectors.toUnmodifiableList()); - - assertEquals(List.of(), applicationPackage.trustedCertificates()); - for (int i = 0; i < certificates.size(); i++) { - applicationPackage = applicationPackage.withTrustedCertificate(certificates.get(i)); - assertEquals(certificates.subList(0, i + 1), applicationPackage.trustedCertificates()); - } - } - - private static byte[] zip(Map<String, String> entries) { - ByteArrayOutputStream zip = new ByteArrayOutputStream(); - try (ZipOutputStream out = new ZipOutputStream(zip)) { - for (Map.Entry<String, String> entry : entries.entrySet()) { - out.putNextEntry(new ZipEntry(entry.getKey())); - out.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); - out.closeEntry(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return zip.toByteArray(); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 640e6860eb6..a464e3d7e9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -264,36 +264,38 @@ public class ApplicationPackageBuilder { xml.append(athenzIdentityAttributes); } xml.append(">\n"); - xml.append(" <instance id='").append(instances).append("'>\n"); - if (upgradePolicy != null || revisionTarget != null || revisionChange != null || upgradeRollout != null) { - xml.append(" <upgrade "); - if (upgradePolicy != null) xml.append("policy='").append(upgradePolicy).append("' "); - if (revisionTarget != null) xml.append("revision-target='").append(revisionTarget).append("' "); - if (revisionChange != null) xml.append("revision-change='").append(revisionChange).append("' "); - if (upgradeRollout != null) xml.append("rollout='").append(upgradeRollout).append("' "); - xml.append("/>\n"); - } - xml.append(notifications); - if (explicitSystemTest) - xml.append(" <test />\n"); - if (explicitStagingTest) - xml.append(" <staging />\n"); - xml.append(blockChange); - xml.append(" <prod"); - if (globalServiceId != null) { - xml.append(" global-service-id='"); - xml.append(globalServiceId); - xml.append("'"); - } - xml.append(">\n"); - xml.append(prodBody); - xml.append(" </prod>\n"); - if (endpointsBody.length() > 0 ) { - xml.append(" <endpoints>\n"); - xml.append(endpointsBody); - xml.append(" </endpoints>\n"); + for (String instance : instances.split(",")) { + xml.append(" <instance id='").append(instance).append("'>\n"); + if (upgradePolicy != null || revisionTarget != null || revisionChange != null || upgradeRollout != null) { + xml.append(" <upgrade "); + if (upgradePolicy != null) xml.append("policy='").append(upgradePolicy).append("' "); + if (revisionTarget != null) xml.append("revision-target='").append(revisionTarget).append("' "); + if (revisionChange != null) xml.append("revision-change='").append(revisionChange).append("' "); + if (upgradeRollout != null) xml.append("rollout='").append(upgradeRollout).append("' "); + xml.append("/>\n"); + } + xml.append(notifications); + if (explicitSystemTest) + xml.append(" <test />\n"); + if (explicitStagingTest) + xml.append(" <staging />\n"); + xml.append(blockChange); + xml.append(" <prod"); + if (globalServiceId != null) { + xml.append(" global-service-id='"); + xml.append(globalServiceId); + xml.append("'"); + } + xml.append(">\n"); + xml.append(prodBody); + xml.append(" </prod>\n"); + if (endpointsBody.length() > 0) { + xml.append(" <endpoints>\n"); + xml.append(endpointsBody); + xml.append(" </endpoints>\n"); + } + xml.append(" </instance>\n"); } - xml.append(" </instance>\n"); if (applicationEndpointsBody.length() > 0) { xml.append(" <endpoints>\n"); xml.append(applicationEndpointsBody); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 989a7c31821..86c21839c96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -32,7 +32,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.application.pkg.ZipStreamReader; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; @@ -41,8 +40,6 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import javax.security.auth.x500.X500Principal; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index e0ce1c060bc..1643f1f818b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -13,6 +13,8 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.net.DomainName; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.text.Text; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -521,7 +523,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer // Returns a canned example response @Override - public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) { + public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, Path restPath) { Map<String,List<?>> root = new HashMap<>(); List<Map<?,?>> resources = new ArrayList<>(); Map<String,String> resource = new HashMap<>(); @@ -532,7 +534,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath) { + public String getServiceStatusPage(DeploymentId deployment, String serviceName, DomainName node, Path subPath) { return "<h1>OK</h1>"; } @@ -567,8 +569,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public ProxyResponse getApplicationPackageContent(DeploymentId deployment, String path, URI requestUri) { - return new ProxyResponse(("{\"path\":\"" + path + "\"}").getBytes(StandardCharsets.UTF_8), "application/json", 200); + public ProxyResponse getApplicationPackageContent(DeploymentId deployment, Path path, URI requestUri) { + return new ProxyResponse(("{\"path\":\"/" + String.join("/", path.segments()) + "\"}").getBytes(StandardCharsets.UTF_8), "application/json", 200); } public void setLogStream(Supplier<String> log) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index a2a1b4ba0a1..7b67db39350 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.TreeMap; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -224,7 +225,7 @@ public class NodeRepositoryMock implements NodeRepository { } public void putApplication(ZoneId zone, Application application) { - applications.computeIfAbsent(zone, (k) -> new HashMap<>()) + applications.computeIfAbsent(zone, (k) -> new TreeMap<>()) .put(application.id(), application); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index c772ca8b8f7..ed00e7f5473 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -18,12 +18,13 @@ import java.time.Duration; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author ogronnesby */ public class CloudTrialExpirerTest { - private final ControllerTester tester = new ControllerTester(SystemName.Public); + private final ControllerTester tester = new ControllerTester(SystemName.PublicCd); private final DeploymentTester deploymentTester = new DeploymentTester(tester); private final CloudTrialExpirer expirer = new CloudTrialExpirer(tester.controller(), Duration.ofMinutes(5)); @@ -35,6 +36,13 @@ public class CloudTrialExpirerTest { } @Test + public void tombstone_inactive_none() { + registerTenant("none-tenant", "none", Duration.ofDays(28).plusMillis(1)); + expirer.maintain(); + assertEquals(Tenant.Type.deleted, tester.controller().tenants().get(TenantName.from("none-tenant"), true).get().type()); + } + + @Test public void keep_inactive_nontrial_tenants() { registerTenant("not-a-trial-tenant", "pay-as-you-go", Duration.ofDays(30)); expirer.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 80ee0988658..10193b48837 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; @@ -38,7 +39,7 @@ public class CostReportMaintainerTest { "1970-01-01,Property3,128.0,96.0,2000.0,0.3333333333333333\n" + "1970-01-01,Property2,160.0,96.0,2000.0,0.3611111111111111", csv), - Map.of(new Property("Property3"), new ResourceAllocation(256, 192, 4000)) + Map.of(new Property("Property3"), new ResourceAllocation(256, 192, 4000, NodeResources.Architecture.getDefault())) ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 4acc9f91e79..6109890bae3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -58,18 +58,18 @@ public class ResourceMeterMaintainerTest { .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().cost().getAsDouble()))); List<ResourceSnapshot> resourceSnapshots = List.of( - new ResourceSnapshot(app1, 12, 34, 56, Instant.EPOCH, z1), - new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), - new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1)); + new ResourceSnapshot(app1, 12, 34, 56, NodeResources.Architecture.getDefault(), Instant.EPOCH, z1), + new ResourceSnapshot(app1, 23, 45, 67, NodeResources.Architecture.getDefault(), Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, NodeResources.Architecture.getDefault(), Instant.EPOCH, z1)); maintainer.updateDeploymentCost(resourceSnapshots); assertCost.accept(app1, Map.of(z1, 1.40, z2, 2.50)); assertCost.accept(app2, Map.of(z1, 3.59)); // Remove a region from app1 and add region to app2 resourceSnapshots = List.of( - new ResourceSnapshot(app1, 23, 45, 67, Instant.EPOCH, z2), - new ResourceSnapshot(app2, 34, 56, 78, Instant.EPOCH, z1), - new ResourceSnapshot(app2, 45, 67, 89, Instant.EPOCH, z2)); + new ResourceSnapshot(app1, 23, 45, 67, NodeResources.Architecture.getDefault(), Instant.EPOCH, z2), + new ResourceSnapshot(app2, 34, 56, 78, NodeResources.Architecture.getDefault(), Instant.EPOCH, z1), + new ResourceSnapshot(app2, 45, 67, 89, NodeResources.Architecture.getDefault(), Instant.EPOCH, z2)); maintainer.updateDeploymentCost(resourceSnapshots); assertCost.accept(app1, Map.of(z2, 2.50)); assertCost.accept(app2, Map.of(z1, 3.59, z2, 4.68)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index d51856b329d..d0dbb23ad1b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; +import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.TenantName; @@ -11,8 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; +import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; +import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import org.junit.Before; import org.junit.Test; @@ -22,6 +29,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; @@ -36,6 +44,19 @@ import static org.junit.Assert.assertTrue; public class NotificationsDbTest { private static final TenantName tenant = TenantName.from("tenant1"); + private static final String email = "user1@example.com"; + private static final CloudTenant cloudTenant = new CloudTenant(tenant, + Instant.now(), + LastLoginInfo.EMPTY, + Optional.empty(), + ImmutableBiMap.of(), + TenantInfo.empty() + .withContacts(new TenantContacts( + List.of(new TenantContacts.EmailContact( + List.of(TenantContacts.Audience.NOTIFICATIONS), + email)))), + List.of(), + Optional.empty()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"), @@ -46,7 +67,8 @@ public class NotificationsDbTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(); - private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb); + private final MockMailer mailer = new MockMailer(); + private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, mailer)); @Test public void list_test() { @@ -75,6 +97,29 @@ public class NotificationsDbTest { } @Test + public void notifier_test() { + Notification notification1 = notification(12345, Type.deployment, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); + Notification notification2 = notification(12345, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3"); + Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg #2"); + + var a = notifications.get(0); + notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages()); + assertEquals(0, mailer.inbox(email).size()); + + // Replace the 3rd notification. but don't change source or type + notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages()); + assertEquals(0, mailer.inbox(email).size()); + + // Notification for a new app, add without replacement + notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages()); + assertEquals(1, mailer.inbox(email).size()); + + // Notification for new type on existing app + notificationsDb.setNotification(notification3.source(), notification3.type(), notification3.level(), notification3.messages()); + assertEquals(2, mailer.inbox(email).size()); + } + + @Test public void remove_single_test() { // Remove the 3rd notification notificationsDb.removeNotification(NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), Type.deployment); @@ -98,6 +143,29 @@ public class NotificationsDbTest { } @Test + public void deployment_metrics_notify_test() { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenant.value(), "app1", "instance1"), ZoneId.from("prod", "us-south-3")); + NotificationSource sourceCluster1 = NotificationSource.from(deploymentId, ClusterSpec.Id.from("cluster1")); + List<Notification> expected = new ArrayList<>(notifications); + + // No metrics, no new notification + notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of()); + assertEquals(0, mailer.inbox(email).size()); + + // Metrics that contain none of the feed block metrics does not create new notification + notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", null, null, null, null, Map.of()))); + assertEquals(0, mailer.inbox(email).size()); + + // One resource is at warning + notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.88, 0.9, 0.3, 0.5, Map.of()))); + assertEquals(1, mailer.inbox(email).size()); + + // One resource over the limit + notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.3, 0.5, Map.of()))); + assertEquals(2, mailer.inbox(email).size()); + } + + @Test public void feed_blocked_single_cluster_test() { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenant.value(), "app1", "instance1"), ZoneId.from("prod", "us-south-3")); NotificationSource sourceCluster1 = NotificationSource.from(deploymentId, ClusterSpec.Id.from("cluster1")); @@ -160,6 +228,8 @@ public class NotificationsDbTest { @Before public void init() { curatorDb.writeNotifications(tenant, notifications); + curatorDb.writeTenant(cloudTenant); + mailer.reset(); } private static List<Notification> notificationIndices(int... indices) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java index 0b28d94386d..e10a41cbae3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -6,6 +6,7 @@ import com.github.tomakehurst.wiremock.stubbing.Scenario; import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.HttpURL.Path; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.protocol.HttpContext; @@ -46,7 +47,7 @@ public class ConfigServerRestExecutorImplTest { stubRequests(path); HttpRequest request = HttpRequest.createTestRequest(url.toString(), com.yahoo.jdisc.http.HttpRequest.Method.GET); - ProxyRequest proxyRequest = ProxyRequest.tryOne(url, path, request); + ProxyRequest proxyRequest = ProxyRequest.tryOne(url, Path.parse(path), request); // Request is retried HttpResponse response = proxy.handle(proxyRequest); @@ -71,7 +72,7 @@ public class ConfigServerRestExecutorImplTest { stubRequests(path); HttpRequest request = HttpRequest.createTestRequest(url.toString(), com.yahoo.jdisc.http.HttpRequest.Method.GET); - ProxyRequest proxyRequest = ProxyRequest.tryOne(url, path, request); + ProxyRequest proxyRequest = ProxyRequest.tryOne(url, Path.parse(path), request); // Connections are reused assertEquals(200, proxy.handle(proxyRequest).getStatus()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index 8542530628c..e8b5df7efa1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.restapi.HttpURL.Path; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -11,20 +12,18 @@ import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; /** * @author Haakon Dybdahl */ public class ProxyRequestTest { - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exception = ExpectedException.none(); - @Test public void testBadUri() { - exception.expectMessage("Request path '/path' does not end with proxy path '/zone/v2/'"); - testRequest("http://domain.tld/path", "/zone/v2/"); + assertThrows("Request path '/path' does not end with proxy path '/zone/v2/'", + IllegalArgumentException.class, + () -> testRequest("http://domain.tld/path", "/zone/v2/")); } @Test @@ -33,7 +32,7 @@ public class ProxyRequestTest { // Root request ProxyRequest request = testRequest("http://controller.domain.tld/my/path", ""); assertEquals(URI.create("http://controller.domain.tld/my/path/"), request.getControllerPrefixUri()); - assertEquals(URI.create("https://cfg.prod.us-north-1.domain.tld:1234/"), + assertEquals(URI.create("https://cfg.prod.us-north-1.domain.tld:1234"), request.createConfigServerRequestUri(URI.create("https://cfg.prod.us-north-1.domain.tld:1234/"))); } @@ -64,7 +63,7 @@ public class ProxyRequestTest { private static ProxyRequest testRequest(String url, String pathPrefix) { return new ProxyRequest(HttpRequest.Method.GET, URI.create(url), Map.of(), null, - List.of(URI.create("http://example.com")), pathPrefix); + List.of(URI.create("http://example.com")), Path.parse(pathPrefix)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 4cc2c1cc79e..1ba0200eec3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.restapi.HttpURL.Path; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -20,7 +21,7 @@ public class ProxyResponseTest { @Test public void testRewriteUrl() throws Exception { ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("http://domain.tld/zone/v2/dev/us-north-1/configserver"), - Map.of(), null, List.of(URI.create("http://example.com")), "configserver"); + Map.of(), null, List.of(URI.create("http://example.com")), Path.parse("configserver")); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", @@ -31,14 +32,14 @@ public class ProxyResponseTest { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); proxyResponse.render(outputStream); - String document = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + String document = outputStream.toString(StandardCharsets.UTF_8); assertEquals("response link is http://domain.tld/zone/v2/dev/us-north-1/bla/bla/", document); } @Test public void testRewriteSecureUrl() throws Exception { ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("https://domain.tld/zone/v2/prod/eu-south-3/configserver"), - Map.of(), null, List.of(URI.create("http://example.com")), "configserver"); + Map.of(), null, List.of(URI.create("http://example.com")), Path.parse("configserver")); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", @@ -49,7 +50,7 @@ public class ProxyResponseTest { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); proxyResponse.render(outputStream); - String document = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + String document = outputStream.toString(StandardCharsets.UTF_8); assertEquals("response link is https://domain.tld/zone/v2/prod/eu-south-3/bla/bla/", document); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index f94f87b0f46..67b201bdc9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -502,7 +503,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Get content/../foo tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/%2E%2E%2Ffoo", GET).userIdentity(USER_ID), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", 403); + accessDenied, 403); // Get content - root tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-east-1/content/", GET).userIdentity(USER_ID), "{\"path\":\"/\"}"); @@ -1069,27 +1070,6 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); } - @Test - public void testMeteringResponses() { - MockMeteringClient mockMeteringClient = tester.serviceRegistry().meteringService(); - - // Mock response for MeteringClient - ResourceAllocation currentSnapshot = new ResourceAllocation(1, 2, 3); - ResourceAllocation thisMonth = new ResourceAllocation(12, 24, 1000); - ResourceAllocation lastMonth = new ResourceAllocation(24, 48, 2000); - ApplicationId applicationId = ApplicationId.from("doesnotexist", "doesnotexist", "default"); - Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory = Map.of(applicationId, List.of( - new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(123), ZoneId.defaultId()), - new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(246), ZoneId.defaultId()), - new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(492), ZoneId.defaultId()))); - - mockMeteringClient.setMeteringData(new MeteringData(thisMonth, lastMonth, currentSnapshot, snapshotHistory)); - - tester.assertResponse(request("/application/v4/tenant/doesnotexist/application/doesnotexist/metering", GET) - .userIdentity(USER_ID) - .oAuthCredentials(OKTA_CREDENTIALS), - new File("instance1-metering.json")); - } @Test public void testRemovingAllDeployments() { @@ -1657,21 +1637,21 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/document/v1/", GET) .userIdentity(USER_ID) .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", - 403); + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at path '/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/document/v1/'\"}", + 404); // Test path traversal tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/../../document/v1/", GET) .userIdentity(USER_ID) .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", - 403); + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at path '/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/document/v1/'\"}", + 404); // Test urlencoded path traversal tester.assertResponse(request(serviceApi + "/storagenode-awe3slno6mmq2fye191y324jl/state%2Fv1%2F..%2F..%2Fdocument%2Fv1%2F", GET) .userIdentity(USER_ID) .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Access denied\"}", + accessDenied, 403); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java index 2c81b1a7fd8..12a0a00713c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java @@ -11,12 +11,12 @@ import org.junit.Test; import java.io.ByteArrayInputStream; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author bratseth @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; public class MultipartParserTest { @Test - public void multipartParserTest() throws URISyntaxException { + public void parser() { String data = "Content-Type: multipart/form-data; boundary=AaB03x\r\n" + "\r\n" + @@ -43,13 +43,7 @@ public class MultipartParserTest { "\r\n" + "... contents of file1.txt ...\r\n" + "--AaB03x--\r\n"; - ByteArrayInputStream dataStream = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)); - HttpRequest request = HttpRequest.createRequest(new MockCurrentContainer(), - new URI("http://foo"), - com.yahoo.jdisc.http.HttpRequest.Method.POST, - dataStream); - request.getJDiscRequest().headers().put("Content-Type", "multipart/form-data; boundary=AaB03x"); - Map<String, byte[]> parts = new MultipartParser().parse(request); + Map<String, byte[]> parts = parse(data, Long.MAX_VALUE); assertEquals(3, parts.size()); assertTrue(parts.containsKey("submit-name")); assertTrue(parts.containsKey("submit-address")); @@ -57,6 +51,41 @@ public class MultipartParserTest { assertEquals("Larry", new String(parts.get("submit-name"), StandardCharsets.UTF_8)); assertEquals("... contents of file1.txt ...", new String(parts.get("files"), StandardCharsets.UTF_8)); } + + @Test + public void max_length() { + String part1 = "Larry"; + String part2 = "House 1"; + String data = + "Content-Type: multipart/form-data; boundary=AaB03x\r\n" + + "\r\n" + + "--AaB03x\r\n" + + "Content-Disposition: form-data; name=\"submit-name\"\r\n" + + "\r\n" + + part1 + "\r\n" + + "--AaB03x\r\n" + + "Content-Disposition: form-data; name=\"submit-address\"\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + + part2 + "\r\n" + + "--AaB03x--\r\n"; + parse(data, part1.length() + part2.length()); + try { + parse(data, part1.length() + part2.length() - 1); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) { + } + } + + private Map<String, byte[]> parse(String data, long maxLength) { + ByteArrayInputStream dataStream = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)); + HttpRequest request = HttpRequest.createRequest(new MockCurrentContainer(), + URI.create("http://foo"), + com.yahoo.jdisc.http.HttpRequest.Method.POST, + dataStream); + request.getJDiscRequest().headers().put("Content-Type", "multipart/form-data; boundary=AaB03x"); + return new MultipartParser(maxLength).parse(request); + } private static class MockCurrentContainer implements CurrentContainer { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java index 9d0054f327a..ce6f713a13d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java @@ -50,6 +50,11 @@ public class ConfigServerApiHandlerTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/configserver/v1"), new File("root.json")); + // GET /configserver/v1/nodes/v2 + tester.assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2"), + "ok"); + assertLastRequest("https://cfg.prod.us-north-1.test.vip:4443/", "GET"); + // GET /configserver/v1/nodes/v2/node/?recursive=true tester.assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node/?recursive=true"), "ok"); @@ -83,12 +88,12 @@ public class ConfigServerApiHandlerTest extends ControllerContainerTest { @Test public void test_allowed_apis() { // GET /configserver/v1/prod/us-north-1 - tester.assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1"), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access '/' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}", + tester.assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/"), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access path '/' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}", 403); tester.assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/application/v2/tenant/vespa"), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access '/application/v2/tenant/vespa' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}", + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access path '/application/v2/tenant/vespa' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}", 403); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 7bc01de2053..8a6244e19a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.test.ManualClock; @@ -158,8 +159,8 @@ public class ControllerApiTest extends ControllerContainerTest { Instant timestamp = Instant.ofEpochMilli(123456789); ZoneId zoneId = ZoneId.defaultId(); List<ResourceSnapshot> snapshots = List.of( - new ResourceSnapshot(applicationId, 12,48,1200, timestamp, zoneId), - new ResourceSnapshot(applicationId, 24, 96,2400, timestamp, zoneId) + new ResourceSnapshot(applicationId, 12,48,1200, NodeResources.Architecture.arm64, timestamp, zoneId), + new ResourceSnapshot(applicationId, 24, 96,2400, NodeResources.Architecture.x86_64, timestamp, zoneId) ); tester.controller().serviceRegistry().meteringService().consume(snapshots); tester.assertResponse( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json index b64e8f26a63..1008ada6def 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json @@ -5,7 +5,8 @@ "zoneId": "prod.default", "cpu": 12.0, "memory": 48.0, - "disk": 1200.0 + "disk": 1200.0, + "architecture":"arm64" }, { "applicationId": "tenant.app.instance", @@ -13,6 +14,7 @@ "zoneId": "prod.default", "cpu": 24.0, "memory": 96.0, - "disk": 2400.0 + "disk": 2400.0, + "architecture":"x86_64" } ]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 7d17e97e66b..cf4deb7b4bf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -125,7 +125,7 @@ public class OsApiTest extends ControllerContainerTest { // Error: Cancel firmware checks in an empty set of zones. assertResponse(new Request("http://localhost:8080/os/v1/firmware/dev/", "", Request.Method.DELETE), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"No zones at path '/os/v1/firmware/dev'\"}", 404); + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No zones at path '/os/v1/firmware/dev/'\"}", 404); assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty()); } diff --git a/dist/vespa.spec b/dist/vespa.spec index e13f06ed0f5..ce2f0137262 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -109,7 +109,7 @@ BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 BuildRequires: vespa-icu-devel >= 65.1.0-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.7.1 +BuildRequires: vespa-onnxruntime-devel = 1.11.0 BuildRequires: vespa-openssl-devel >= 1.1.1n-1 %define _use_vespa_openssl 1 BuildRequires: vespa-protobuf-devel = 3.19.1 @@ -137,7 +137,7 @@ BuildRequires: vespa-openssl-devel >= 1.1.1n-1 BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.7.1 +BuildRequires: vespa-onnxruntime-devel = 1.11.0 BuildRequires: vespa-protobuf-devel = 3.19.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif @@ -146,7 +146,7 @@ BuildRequires: cmake >= 3.20.2 BuildRequires: maven BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.7.1 +BuildRequires: vespa-onnxruntime-devel = 1.11.0 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 BuildRequires: protobuf-devel BuildRequires: (llvm-devel >= 13.0.0 and llvm-devel < 14) @@ -159,7 +159,7 @@ BuildRequires: cmake >= 3.9.1 BuildRequires: maven BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.7.1 +BuildRequires: vespa-onnxruntime-devel = 1.11.0 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %if 0%{?fc34} BuildRequires: protobuf-devel @@ -450,7 +450,7 @@ Requires: llvm-libs >= 13.0.1 Requires: llvm-libs >= 13.0.1 %endif %endif -Requires: vespa-onnxruntime = 1.7.1 +Requires: vespa-onnxruntime = 1.11.0 %description libs diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 4668942b61e..1fe6d4cc86c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -21,12 +21,13 @@ import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 class MessageFactory { private final static Logger log = Logger.getLogger(MessageFactory.class.getName()); private final Message requestMsg; - private final LoadType loadType; - private final DocumentProtocol.Priority priority; + private final LoadType loadType; // TODO: Remove on Vespa 8 + private final DocumentProtocol.Priority priority; // TODO: Remove on Vespa 8 @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public MessageFactory(DocumentMessage requestMsg) { diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java index de96c548652..51b068b4712 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.objects.Ids; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.OptionalInt; +import java.util.Objects; /** * A StringFieldValue is a wrapper class that holds a String in {@link com.yahoo.document.Document}s and @@ -57,10 +57,9 @@ public class StringFieldValue extends FieldValue { } private static void validateTextString(String value) { - OptionalInt illegalCodePoint = Text.validateTextString(value); - if (illegalCodePoint.isPresent()) { + if ( ! Text.isValidTextString(value)) { throw new IllegalArgumentException("The string field value contains illegal code point 0x" + - Integer.toHexString(illegalCodePoint.getAsInt()).toUpperCase()); + Integer.toHexString(Text.validateTextString(value).getAsInt()).toUpperCase()); } } @@ -88,7 +87,7 @@ public class StringFieldValue extends FieldValue { public StringFieldValue clone() { StringFieldValue strfval = (StringFieldValue) super.clone(); if (spanTrees != null) { - strfval.spanTrees = new HashMap<String, SpanTree>(spanTrees.size()); + strfval.spanTrees = new HashMap<>(spanTrees.size()); for (Map.Entry<String, SpanTree> entry : spanTrees.entrySet()) { strfval.spanTrees.put(entry.getKey(), new SpanTree(entry.getValue())); } @@ -240,8 +239,8 @@ public class StringFieldValue extends FieldValue { if (!(o instanceof StringFieldValue)) return false; if (!super.equals(o)) return false; StringFieldValue that = (StringFieldValue) o; - if ((spanTrees != null) ? !spanTrees.equals(that.spanTrees) : that.spanTrees != null) return false; - if ((value != null) ? !value.equals(that.value) : that.value != null) return false; + if (!Objects.equals(spanTrees, that.spanTrees)) return false; + if (!Objects.equals(value, that.value)) return false; return true; } diff --git a/document/src/main/java/com/yahoo/document/idstring/IdString.java b/document/src/main/java/com/yahoo/document/idstring/IdString.java index 55beff9eef9..40ac19dec6d 100644 --- a/document/src/main/java/com/yahoo/document/idstring/IdString.java +++ b/document/src/main/java/com/yahoo/document/idstring/IdString.java @@ -5,8 +5,6 @@ import com.yahoo.api.annotations.Beta; import com.yahoo.text.Text; import com.yahoo.text.Utf8String; -import java.util.OptionalInt; - /** * To be used with DocumentId constructor. * @@ -81,10 +79,9 @@ public abstract class IdString { } private static void validateTextString(String id) { - OptionalInt illegalCodePoint = Text.validateTextString(id); - if (illegalCodePoint.isPresent()) { + if ( ! Text.isValidTextString(id)) { throw new IllegalArgumentException("Unparseable id '" + id + "': Contains illegal code point 0x" + - Integer.toHexString(illegalCodePoint.getAsInt()).toUpperCase()); + Integer.toHexString(Text.validateTextString(id).getAsInt()).toUpperCase()); } } diff --git a/document/src/tests/repo/doctype_config_test.cpp b/document/src/tests/repo/doctype_config_test.cpp index 84ec1414fcc..eab40e04617 100644 --- a/document/src/tests/repo/doctype_config_test.cpp +++ b/document/src/tests/repo/doctype_config_test.cpp @@ -659,4 +659,14 @@ TEST("Tensor fields have tensor types") { EXPECT_TRUE(&tensorField1.getDataType() == tensorFieldValue1->getDataType()); } +TEST("requireThatImportedFieldsWorks") { + DocumentTypeRepo repo(readDocumenttypesConfig(TEST_PATH("import-dt.cfg"))); + ASSERT_TRUE(repo.getDocumentType("document")); + ASSERT_TRUE(repo.getDocumentType("grandparent")); + ASSERT_TRUE(repo.getDocumentType("parent_a")); + ASSERT_TRUE(repo.getDocumentType("parent_b")); + ASSERT_TRUE(repo.getDocumentType("child")); +} + + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/document/src/tests/repo/import-dt.cfg b/document/src/tests/repo/import-dt.cfg new file mode 100644 index 00000000000..742f377e65c --- /dev/null +++ b/document/src/tests/repo/import-dt.cfg @@ -0,0 +1,139 @@ +enablecompression false +usev8geopositions false +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "child" +doctype[1].idx 10015 +doctype[1].internalid 746267614 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{myfieldset}.fields[0] "my_ancient_int_field" +doctype[1].fieldsets{myfieldset}.fields[1] "my_int_field" +doctype[1].fieldsets{myfieldset}.fields[2] "my_string_field" +doctype[1].fieldsets{[document]}.fields[0] "a_ref" +doctype[1].fieldsets{[document]}.fields[1] "b_ref" +doctype[1].fieldsets{[document]}.fields[2] "b_ref_with_summary" +doctype[1].importedfield[0].name "my_int_field" +doctype[1].importedfield[1].name "my_string_field" +doctype[1].importedfield[2].name "my_int_array_field" +doctype[1].importedfield[3].name "my_int_wset_field" +doctype[1].importedfield[4].name "my_ancient_int_field" +doctype[1].documentref[0].idx 10017 +doctype[1].documentref[0].targettype 10018 +doctype[1].documentref[0].internalid -1586898847 +doctype[1].documentref[1].idx 10019 +doctype[1].documentref[1].targettype 10020 +doctype[1].documentref[1].internalid -1586898816 +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "child.header" +doctype[1].structtype[0].field[0].name "a_ref" +doctype[1].structtype[0].field[0].internalid 16961427 +doctype[1].structtype[0].field[0].type 10017 +doctype[1].structtype[0].field[1].name "b_ref" +doctype[1].structtype[0].field[1].internalid 590403512 +doctype[1].structtype[0].field[1].type 10019 +doctype[1].structtype[0].field[2].name "b_ref_with_summary" +doctype[1].structtype[0].field[2].internalid 232360236 +doctype[1].structtype[0].field[2].type 10019 +doctype[1].structtype[0].internalid 81425825 +doctype[2].name "grandparent" +doctype[2].idx 10021 +doctype[2].internalid -154107656 +doctype[2].inherits[0].idx 10000 +doctype[2].contentstruct 10022 +doctype[2].fieldsets{[document]}.fields[0] "int_field" +doctype[2].structtype[0].idx 10022 +doctype[2].structtype[0].name "grandparent.header" +doctype[2].structtype[0].field[0].name "int_field" +doctype[2].structtype[0].field[0].internalid 2128822283 +doctype[2].structtype[0].field[0].type 10007 +doctype[2].structtype[0].internalid 990971719 +doctype[3].name "parent_a" +doctype[3].idx 10018 +doctype[3].internalid -244366130 +doctype[3].inherits[0].idx 10000 +doctype[3].contentstruct 10023 +doctype[3].fieldsets{[document]}.fields[0] "grandparent_ref" +doctype[3].fieldsets{[document]}.fields[1] "int_array_field" +doctype[3].fieldsets{[document]}.fields[2] "int_field" +doctype[3].fieldsets{[document]}.fields[3] "int_wset_field" +doctype[3].importedfield[0].name "ancient_int_field" +doctype[3].arraytype[0].idx 10025 +doctype[3].arraytype[0].elementtype 10007 +doctype[3].arraytype[0].internalid -1245117006 +doctype[3].wsettype[0].idx 10026 +doctype[3].wsettype[0].elementtype 10007 +doctype[3].wsettype[0].createifnonexistent false +doctype[3].wsettype[0].removeifzero false +doctype[3].wsettype[0].internalid 519906144 +doctype[3].documentref[0].idx 10024 +doctype[3].documentref[0].targettype 10021 +doctype[3].documentref[0].internalid -1714181319 +doctype[3].structtype[0].idx 10023 +doctype[3].structtype[0].name "parent_a.header" +doctype[3].structtype[0].field[0].name "grandparent_ref" +doctype[3].structtype[0].field[0].internalid 29565679 +doctype[3].structtype[0].field[0].type 10024 +doctype[3].structtype[0].field[1].name "int_field" +doctype[3].structtype[0].field[1].internalid 2128822283 +doctype[3].structtype[0].field[1].type 10007 +doctype[3].structtype[0].field[2].name "int_array_field" +doctype[3].structtype[0].field[2].internalid 85807681 +doctype[3].structtype[0].field[2].type 10025 +doctype[3].structtype[0].field[3].name "int_wset_field" +doctype[3].structtype[0].field[3].internalid 1945161474 +doctype[3].structtype[0].field[3].type 10026 +doctype[3].structtype[0].internalid 236742321 +doctype[4].name "parent_b" +doctype[4].idx 10020 +doctype[4].internalid -244365169 +doctype[4].inherits[0].idx 10000 +doctype[4].contentstruct 10027 +doctype[4].fieldsets{[document]}.fields[0] "string_field" +doctype[4].structtype[0].idx 10027 +doctype[4].structtype[0].name "parent_b.header" +doctype[4].structtype[0].field[0].name "string_field" +doctype[4].structtype[0].field[0].internalid 1222457840 +doctype[4].structtype[0].field[0].type 10012 +doctype[4].structtype[0].internalid 40228816 diff --git a/document/src/vespa/document/base/fieldpath.cpp b/document/src/vespa/document/base/fieldpath.cpp index fcac59847cb..01855af55eb 100644 --- a/document/src/vespa/document/base/fieldpath.cpp +++ b/document/src/vespa/document/base/fieldpath.cpp @@ -11,7 +11,6 @@ using vespalib::make_string; namespace document { -FieldPathEntry::FieldPathEntry(const FieldPathEntry &) = default; FieldPathEntry::~FieldPathEntry() = default; FieldPathEntry::FieldPathEntry() : @@ -46,7 +45,7 @@ FieldPathEntry::FieldPathEntry(const Field &fieldRef) : _lookupIndex(0), _lookupKey(), _variableName(), - _fillInVal(fieldRef.createValue().release()) + _fillInVal(fieldRef.createValue()) { } FieldPathEntry::FieldPathEntry(const DataType & dataType, const DataType& fillType, @@ -56,13 +55,24 @@ FieldPathEntry::FieldPathEntry(const DataType & dataType, const DataType& fillTy _field(), _dataType(&dataType), _lookupIndex(0), - _lookupKey(lookupKey.release()), + _lookupKey(std::move(lookupKey)), _variableName(), _fillInVal() { setFillValue(fillType); } +FieldPathEntry::FieldPathEntry(const FieldPathEntry &rhs) + : _type(rhs._type), + _name(rhs._name), + _field(rhs._field), + _dataType(rhs._dataType), + _lookupIndex(rhs._lookupIndex), + _lookupKey(rhs._lookupKey ? rhs._lookupKey->clone() : nullptr), + _variableName(rhs._variableName), + _fillInVal(rhs._fillInVal ? rhs._fillInVal->clone() : nullptr) +{} + void FieldPathEntry::setFillValue(const DataType & dataType) { @@ -82,7 +92,7 @@ FieldPathEntry::setFillValue(const DataType & dataType) } } if (dt->isPrimitive()) { - _fillInVal.reset(dt->createFieldValue().release()); + _fillInVal = dt->createFieldValue(); } } @@ -123,7 +133,7 @@ FieldPathEntry::getDataType() const FieldValue::UP FieldPathEntry::stealFieldValueToSet() const { - return FieldValue::UP(_fillInVal.release()); + return std::move(_fillInVal); } vespalib::string @@ -171,17 +181,20 @@ FieldPathEntry::parseKey(vespalib::stringref & key) return v; } -FieldPath::FieldPath() - : _path() -{ } - -FieldPath::FieldPath(const FieldPath &) = default; -FieldPath & FieldPath::operator=(const FieldPath &) = default; +FieldPath::FieldPath() = default; FieldPath::~FieldPath() = default; +FieldPath::FieldPath(const FieldPath & rhs) + : _path() +{ + _path.reserve(rhs.size()); + for (const auto & e : rhs._path) { + _path.emplace_back(std::make_unique<FieldPathEntry>(*e)); + } +} FieldPath::iterator FieldPath::insert(iterator pos, std::unique_ptr<FieldPathEntry> entry) { - return _path.insert(pos, vespalib::CloneablePtr<FieldPathEntry>(entry.release())); + return _path.insert(pos, std::move(entry)); } void FieldPath::push_back(std::unique_ptr<FieldPathEntry> entry) { _path.emplace_back(entry.release()); } void FieldPath::pop_back() { _path.pop_back(); } diff --git a/document/src/vespa/document/base/fieldpath.h b/document/src/vespa/document/base/fieldpath.h index a79e4595a61..95f916af118 100644 --- a/document/src/vespa/document/base/fieldpath.h +++ b/document/src/vespa/document/base/fieldpath.h @@ -2,7 +2,6 @@ #pragma once #include "field.h" -#include <vespa/vespalib/util/memory.h> namespace document { @@ -23,13 +22,12 @@ public: VARIABLE, NONE }; - using FieldValueCP = vespalib::CloneablePtr<FieldValue>; - FieldPathEntry(); - FieldPathEntry(FieldPathEntry &&) = default; - FieldPathEntry & operator=(FieldPathEntry &&) = default; + FieldPathEntry(FieldPathEntry &&) noexcept = default; + FieldPathEntry & operator=(FieldPathEntry &&) noexcept = default; FieldPathEntry(const FieldPathEntry &); + FieldPathEntry & operator=(const FieldPathEntry &) = delete; /** Creates a field path entry for a struct field lookup. @@ -58,8 +56,6 @@ public: */ FieldPathEntry(const DataType & dataType, vespalib::stringref variableName); - FieldPathEntry * clone() const { return new FieldPathEntry(*this); } - Type getType() const { return _type; } const vespalib::string & getName() const { return _name; } @@ -70,7 +66,7 @@ public: uint32_t getIndex() const { return _lookupIndex; } - const FieldValueCP & getLookupKey() const { return _lookupKey; } + const FieldValue & getLookupKey() const { return *_lookupKey; } const vespalib::string& getVariableName() const { return _variableName; } @@ -85,20 +81,20 @@ public: static vespalib::string parseKey(vespalib::stringref & key); private: void setFillValue(const DataType & dataType); - Type _type; - vespalib::string _name; - Field _field; - const DataType * _dataType; - uint32_t _lookupIndex; - FieldValueCP _lookupKey; - vespalib::string _variableName; - mutable FieldValueCP _fillInVal; + Type _type; + vespalib::string _name; + Field _field; + const DataType * _dataType; + uint32_t _lookupIndex; + std::unique_ptr<FieldValue> _lookupKey; + vespalib::string _variableName; + mutable std::unique_ptr<FieldValue> _fillInVal; }; //typedef std::deque<FieldPathEntry> FieldPath; // Facade over FieldPathEntry container that exposes cloneability class FieldPath { - typedef std::vector<vespalib::CloneablePtr<FieldPathEntry>> Container; + typedef std::vector<std::unique_ptr<FieldPathEntry>> Container; public: typedef Container::reference reference; typedef Container::const_reference const_reference; @@ -110,16 +106,11 @@ public: FieldPath(); FieldPath(const FieldPath &); - FieldPath & operator=(const FieldPath &); + FieldPath & operator=(const FieldPath &) = delete; FieldPath(FieldPath &&) noexcept = default; FieldPath & operator=(FieldPath &&) noexcept = default; ~FieldPath(); - template <typename InputIterator> - FieldPath(InputIterator first, InputIterator last) - : _path(first, last) - { } - iterator insert(iterator pos, std::unique_ptr<FieldPathEntry> entry); void push_back(std::unique_ptr<FieldPathEntry> entry); diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.h b/document/src/vespa/document/fieldvalue/iteratorhandler.h index bef013e1845..2f7b5d522e9 100644 --- a/document/src/vespa/document/fieldvalue/iteratorhandler.h +++ b/document/src/vespa/document/fieldvalue/iteratorhandler.h @@ -77,7 +77,7 @@ public: void setArrayIndex(uint32_t index) { _arrayIndexStack.back() = index; } ModificationStatus modify(FieldValue &fv) { return doModify(fv); } fieldvalue::VariableMap &getVariables() { return _variables; } - void setVariables(const fieldvalue::VariableMap &vars) { _variables = vars; } + void setVariables(fieldvalue::VariableMap vars) { _variables = std::move(vars); } virtual bool createMissingPath() const { return false; } private: virtual bool onComplex(const Content &fv) { diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp index 33ce4357f03..7385b261339 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp @@ -421,9 +421,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested, case FieldPathEntry::MAP_KEY: { LOG(spam, "MAP_KEY"); - const_iterator iter = find(*fpe.getLookupKey()); + const_iterator iter = find(fpe.getLookupKey()); if (iter != end()) { - wasModified = checkAndRemove(*fpe.getLookupKey(), + wasModified = checkAndRemove(fpe.getLookupKey(), iter->second->iterateNested(nested.next(), handler), wasModified, keysToRemove); } else if (handler.createMissingPath()) { @@ -431,7 +431,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested, FieldValue::UP val = getMapType().getValueType().createFieldValue(); ModificationStatus status = val->iterateNested(nested.next(), handler); if (status == ModificationStatus::MODIFIED) { - const_cast<MapFieldValue&>(*this).put(FieldValue::UP(fpe.getLookupKey()->clone()), std::move(val)); + const_cast<MapFieldValue&>(*this).put(FieldValue::UP(fpe.getLookupKey().clone()), std::move(val)); return status; } } diff --git a/document/src/vespa/document/fieldvalue/variablemap.cpp b/document/src/vespa/document/fieldvalue/variablemap.cpp index 481e5288a24..1ad4e1b716b 100644 --- a/document/src/vespa/document/fieldvalue/variablemap.cpp +++ b/document/src/vespa/document/fieldvalue/variablemap.cpp @@ -11,8 +11,8 @@ IndexValue::IndexValue(int index_) : index(index_), key() {} bool IndexValue::operator==(const IndexValue& other) const { - if (key.get() != NULL) { - if (other.key.get() != NULL && *key == *other.key) { + if (key) { + if (other.key && *key == *other.key) { return true; } return false; @@ -22,28 +22,32 @@ IndexValue::operator==(const IndexValue& other) const { } IndexValue::IndexValue(const FieldValue& key_) - : index(-1), - key(FieldValue::CP(key_.clone())) + : index(-1), + key(key_.clone()) { } +IndexValue::IndexValue(IndexValue && rhs) noexcept = default; +IndexValue & IndexValue::operator = (IndexValue && rhs) noexcept = default; IndexValue::IndexValue(const IndexValue & rhs) = default; IndexValue & IndexValue::operator = (const IndexValue & rhs) = default; -IndexValue::~IndexValue() { } +IndexValue::~IndexValue() = default; vespalib::string IndexValue::toString() const { - if (key.get() != NULL) { + if (key) { return key->toString(); } else { return vespalib::make_string("%d", index); } } -VariableMap::VariableMap() {} -VariableMap::~VariableMap() {} +VariableMap::VariableMap(VariableMap && rhs) noexcept = default; +VariableMap & VariableMap::operator = (VariableMap && rhs) noexcept = default; VariableMap::VariableMap(const VariableMap & rhs) = default; VariableMap & VariableMap::operator = (const VariableMap & rhs) = default; +VariableMap::VariableMap() = default; +VariableMap::~VariableMap() = default; vespalib::string VariableMap::toString() const { diff --git a/document/src/vespa/document/fieldvalue/variablemap.h b/document/src/vespa/document/fieldvalue/variablemap.h index 70dba1551ef..68b8bcf09e8 100644 --- a/document/src/vespa/document/fieldvalue/variablemap.h +++ b/document/src/vespa/document/fieldvalue/variablemap.h @@ -17,8 +17,8 @@ public: IndexValue(); IndexValue(int index_); IndexValue(const FieldValue& key_); - IndexValue(IndexValue && rhs) = default; - IndexValue & operator = (IndexValue && rhs) = default; + IndexValue(IndexValue && rhs) noexcept; + IndexValue & operator = (IndexValue && rhs) noexcept; IndexValue(const IndexValue & rhs); IndexValue & operator = (const IndexValue & rhs); @@ -36,8 +36,8 @@ using VariableMapT = std::map<vespalib::string, IndexValue>; class VariableMap : public VariableMapT { public: VariableMap(); - VariableMap(VariableMap && rhs) = default; - VariableMap & operator = (VariableMap && rhs) = default; + VariableMap(VariableMap && rhs) noexcept; + VariableMap & operator = (VariableMap && rhs) noexcept; VariableMap(const VariableMap & rhs); VariableMap & operator = (const VariableMap & rhs); ~VariableMap(); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index d8f272d5d55..147e79bbf32 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -37,12 +37,20 @@ namespace document { namespace internal { -using DocumentTypeMapT = vespalib::hash_map<int32_t, DataTypeRepo *>; +using DocumentTypeMapT = std::map<int32_t, std::unique_ptr<DataTypeRepo>>; class DocumentTypeMap : public DocumentTypeMapT { public: using DocumentTypeMapT::DocumentTypeMapT; + DataTypeRepo * findRepo(int32_t doc_type_id) const { + auto iter = find(doc_type_id); + if (iter == end()) { + return nullptr; + } else { + return iter->second.get(); + } + } }; } @@ -50,28 +58,16 @@ public: using DocumentTypeMap = internal::DocumentTypeMap; namespace { -template <typename Container> -void DeleteContent(Container &c) { - for (auto ptr : c) { - delete ptr; - } -} -template <typename Map> -void DeleteMapContent(Map &m) { - for (auto & entry : m) { - delete entry.second; - } -} // A collection of data types. class Repo { - vector<const DataType *> _owned_types; - hash_map<int32_t, const DataType *> _types; + vector<std::unique_ptr<const DataType>> _owned_types; + hash_map<int32_t, const DataType *> _id_map; hash_map<string, const DataType *> _tensorTypes; hash_map<string, const DataType *> _name_map; public: - ~Repo() { DeleteContent(_owned_types); } + ~Repo() {} void inherit(const Repo &parent); bool addDataType(const DataType &type); @@ -85,14 +81,14 @@ public: }; void Repo::inherit(const Repo &parent) { - _types.insert(parent._types.begin(), parent._types.end()); + _id_map.insert(parent._id_map.begin(), parent._id_map.end()); _tensorTypes.insert(parent._tensorTypes.begin(), parent._tensorTypes.end()); _name_map.insert(parent._name_map.begin(), parent._name_map.end()); } // Returns true if a reference to type is stored. bool Repo::addDataType(const DataType &type) { - const DataType *& data_type = _types[type.getId()]; + const DataType *& data_type = _id_map[type.getId()]; if (data_type) { if (data_type->equals(type) && (data_type->getName() == type.getName())) { return false; // Redefinition of identical type is ok. @@ -117,9 +113,9 @@ template <typename T> const DataType* Repo::addDataType(unique_ptr<T> type) { int id = type->getId(); if (addDataType(*type)) { - _owned_types.push_back(type.release()); + _owned_types.emplace_back(std::move(type)); } - return _types[id]; + return _id_map[id]; } @@ -127,28 +123,21 @@ const DataType & Repo::addTensorType(const string &spec) { auto type = TensorDataType::fromSpec(spec); - auto insres = _tensorTypes.insert(std::make_pair(spec, type.get())); - if (insres.second) { - _owned_types.push_back(type.release()); + auto [ iter, inserted ] = _tensorTypes.insert(std::make_pair(spec, type.get())); + if (inserted) { + _owned_types.emplace_back(std::move(type)); } - return *insres.first->second; -} - -template <typename Map> -typename Map::mapped_type FindPtr(const Map &m, typename Map::key_type key) { - typename Map::const_iterator it = m.find(key); - if (it != m.end()) { - return it->second; - } - return typename Map::mapped_type(); + return *iter->second; } const DataType *Repo::lookup(int32_t id) const { - return FindPtr(_types, id); + auto iter = _id_map.find(id); + return (iter == _id_map.end()) ? nullptr : iter->second; } const DataType *Repo::lookup(stringref n) const { - return FindPtr(_name_map, n); + auto iter = _name_map.find(n); + return (iter == _name_map.end()) ? nullptr : iter->second; } const DataType &Repo::findOrThrow(int32_t id) const { @@ -169,11 +158,11 @@ Repo::findOrThrowOrCreate(int32_t id, const string &detailedType) } class AnnotationTypeRepo { - vector<const AnnotationType *> _owned_types; + vector<std::unique_ptr<const AnnotationType>> _owned_types; hash_map<int32_t, AnnotationType *> _annotation_types; public: - ~AnnotationTypeRepo() { DeleteContent(_owned_types); } + ~AnnotationTypeRepo() {} void inherit(const AnnotationTypeRepo &parent); AnnotationType * addAnnotationType(AnnotationType::UP annotation_type); @@ -196,7 +185,7 @@ AnnotationType * AnnotationTypeRepo::addAnnotationType(AnnotationType::UP type) } } else { a_type = type.get(); - _owned_types.push_back(type.release()); + _owned_types.emplace_back(std::move(type)); } return a_type; } @@ -215,7 +204,8 @@ void AnnotationTypeRepo::setAnnotationDataType(int32_t id, const DataType &d) { } const AnnotationType *AnnotationTypeRepo::lookup(int32_t id) const { - return FindPtr(_annotation_types, id); + auto iter = _annotation_types.find(id); + return (iter == _annotation_types.end()) ? nullptr : iter->second; } } // namespace @@ -225,12 +215,14 @@ const AnnotationType *AnnotationTypeRepo::lookup(int32_t id) const { struct DataTypeRepo { typedef unique_ptr<DataTypeRepo> UP; - DocumentType *doc_type; + std::unique_ptr<DocumentType> doc_type; Repo repo; AnnotationTypeRepo annotations; - DataTypeRepo() : doc_type(nullptr) {} - ~DataTypeRepo() { delete doc_type; } + DataTypeRepo() : doc_type() {} + ~DataTypeRepo() {} + + DocumentType * doc() const { return doc_type.get(); } }; namespace { @@ -363,13 +355,15 @@ void addDataTypes(const vector<Datatype> &types, Repo &repo, const AnnotationTyp } void addDocumentTypes(const DocumentTypeMap &type_map, Repo &repo) { - for (const auto & entry : type_map) { - repo.addDataType(*entry.second->doc_type); + for (const auto & [ key, data_type_repo ] : type_map) { + repo.addDataType(*data_type_repo->doc()); } } const DocumentType * addDefaultDocument(DocumentTypeMap &type_map) { + const uint32_t typeId = DataType::T_DOCUMENT; + auto data_types = std::make_unique<DataTypeRepo>(); vector<const DataType *> default_types = DataType::getDefaultDataTypes(); for (size_t i = 0; i < default_types.size(); ++i) { @@ -377,25 +371,23 @@ addDefaultDocument(DocumentTypeMap &type_map) { } data_types->repo.addDataType(UrlDataType::getInstance()); data_types->repo.addDataType(PositionDataType::getInstance()); - data_types->doc_type = new DocumentType("document", 8); + data_types->doc_type = std::make_unique<DocumentType>("document", typeId); vector<const AnnotationType *> annotation_types(AnnotationType::getDefaultAnnotationTypes()); for(size_t i(0); i < annotation_types.size(); ++i) { data_types->annotations.addAnnotationType(std::make_unique<AnnotationType>(*annotation_types[i])); } - - uint32_t typeId = data_types->doc_type->getId(); - const DocumentType * docType = data_types->doc_type; - type_map[typeId] = data_types.release(); + const DocumentType * docType = data_types->doc(); + type_map[typeId] = std::move(data_types); return docType; } const DataTypeRepo &lookupRepo(int32_t id, const DocumentTypeMap &type_map) { - DocumentTypeMap::const_iterator it = type_map.find(id); - if (it == type_map.end()) { + if (const auto * p = type_map.findRepo(id)) { + return *p; + } else { throw IllegalArgumentException(make_string("Unable to find document type %d.", id)); } - return *it->second; } void inheritDataTypes(const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types, @@ -427,7 +419,7 @@ makeDataTypeRepo(const DocumentType &doc_type, const DocumentTypeMap &type_map) auto data_types = std::make_unique<DataTypeRepo>(); data_types->repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).repo); data_types->annotations.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).annotations); - data_types->doc_type = new DocumentType(doc_type); + data_types->doc_type = std::make_unique<DocumentType>(doc_type); return data_types; } @@ -446,8 +438,8 @@ void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Reference Repo& data_type_repo, const DocumentTypeMap& doc_type_map) { for (const auto& ref_type : ref_types) { - const auto* target_doc_type = lookupRepo(ref_type.targetTypeId, doc_type_map).doc_type; - data_type_repo.addDataType(std::make_unique<ReferenceDataType>(*target_doc_type, ref_type.id)); + const auto & repo = lookupRepo(ref_type.targetTypeId, doc_type_map); + data_type_repo.addDataType(std::make_unique<ReferenceDataType>(*repo.doc_type, ref_type.id)); } } @@ -460,31 +452,31 @@ void add_imported_fields(const DocumenttypesConfig::Documenttype::ImportedfieldV } void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) { - DataTypeRepo *data_types = type_map[doc_type.id]; + const auto & data_types = type_map[doc_type.id]; inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations); addAnnotationTypes(doc_type.annotationtype, data_types->annotations); inheritDataTypes(doc_type.inherits, type_map, data_types->repo); addReferenceTypes(doc_type.referencetype, data_types->repo, type_map); addDataTypes(doc_type.datatype, data_types->repo, data_types->annotations); setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo); - inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type); - addFieldSet(doc_type.fieldsets, *data_types->doc_type); - add_imported_fields(doc_type.importedfield, *data_types->doc_type); + inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc()); + addFieldSet(doc_type.fieldsets, *data_types->doc()); + add_imported_fields(doc_type.importedfield, *data_types->doc()); } void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) { - DataTypeRepo *& p = doc_types[data_types->doc_type->getId()]; + auto & p = doc_types[data_types->doc()->getId()]; if (p) { - LOG(warning, "Type repo already exists for id %d.", data_types->doc_type->getId()); + LOG(warning, "Type repo already exists for id %d.", data_types->doc()->getId()); throw IllegalArgumentException("Trying to redefine a document type."); } - p = data_types.release(); + p = std::move(data_types); } DataTypeRepo::UP makeSkeletonDataTypeRepo(const DocumenttypesConfig::Documenttype &type) { auto data_types = std::make_unique<DataTypeRepo>(); auto type_ap = std::make_unique<StructDataType>(type.name + ".header", type.headerstruct); - data_types->doc_type = new DocumentType(type.name, type.id, *type_ap); + data_types->doc_type = std::make_unique<DocumentType>(type.name, type.id, *type_ap); data_types->repo.addDataType(std::move(type_ap)); return data_types; } @@ -534,22 +526,23 @@ private: struct DocTypeInProgress { const CDocType & cfg; - DataTypeRepo * data_type_repo; - DocumentType * dtype = nullptr; + DataTypeRepo * data_type_repo = nullptr; bool builtin = false; DocTypeInProgress(const CDocType & config, DocumentTypeMap &doc_types) - : cfg(config), - data_type_repo(doc_types[cfg.internalid]) + : cfg(config) { - if (data_type_repo) { + auto iter = doc_types.find(cfg.internalid); + if (iter != doc_types.end()) { LOG(debug, "old doct : %s [%d]", cfg.name.c_str(), cfg.internalid); builtin = true; } else { LOG(debug, "new doct : %s [%d]", cfg.name.c_str(), cfg.internalid); - data_type_repo = new DataTypeRepo(); - doc_types[cfg.internalid] = data_type_repo; + doc_types.emplace(cfg.internalid, std::make_unique<DataTypeRepo>()); } + iter = doc_types.find(cfg.internalid); + LOG_ASSERT(iter != doc_types.end()); + data_type_repo = iter->second.get(); } Repo& repo() { return data_type_repo->repo; } @@ -587,14 +580,13 @@ private: createEmptyStructs(dtInP); initializeDocTypeAndInheritAnnotations(dtInP); createEmptyAnnotationTypes(dtInP); + } + for (auto & [id, dtInP] : _doc_types_in_progress) { createReferenceTypes(dtInP); } createComplexTypes(); fillStructs(); - for (const CDocType & docT : _input) { - auto iter = _doc_types_in_progress.find(docT.idx); - LOG_ASSERT(iter != _doc_types_in_progress.end()); - auto & dtInP = iter->second; + for (auto & [id, dtInP] : _doc_types_in_progress) { fillDocument(dtInP); fillAnnotationTypes(dtInP); } @@ -674,15 +666,15 @@ private: void initializeDocTypeAndInheritAnnotations(DocTypeInProgress & dtInP) { if (dtInP.builtin) { - madeType(dtInP.data_type_repo->doc_type, dtInP.cfg.idx); + madeType(dtInP.data_type_repo->doc(), dtInP.cfg.idx); return; } - LOG_ASSERT(dtInP.data_type_repo->doc_type == nullptr); + LOG_ASSERT(dtInP.data_type_repo->doc() == nullptr); const auto & docT = dtInP.cfg; const StructDataType * fields = findStruct(docT.contentstruct); if (fields != nullptr) { - dtInP.data_type_repo->doc_type = new DocumentType(docT.name, docT.internalid, *fields); - madeType(dtInP.data_type_repo->doc_type, docT.idx); + dtInP.data_type_repo->doc_type = std::make_unique<DocumentType>(docT.name, docT.internalid, *fields); + madeType(dtInP.data_type_repo->doc(), docT.idx); } else { LOG(error, "Missing content struct for '%s' (idx %d not found)", docT.name.c_str(), docT.contentstruct); @@ -696,7 +688,7 @@ private: inheritD.idx, docT.name.c_str()); throw IllegalArgumentException("Unable to find document for inheritance"); } - DataTypeRepo * parentRepo = FindPtr(_output, dt->getId()); + const DataTypeRepo *parentRepo = _output.findRepo(dt->getId()); if (parentRepo == nullptr) { LOG(error, "parent repo [id %d] missing for document %s", dt->getId(), docT.name.c_str()); @@ -822,7 +814,7 @@ private: return; } const CDocType & docT = dtInP.cfg; - auto * doc_type = dtInP.data_type_repo->doc_type; + auto * doc_type = dtInP.data_type_repo->doc(); LOG_ASSERT(doc_type != nullptr); for (const auto & importD : docT.importedfield) { doc_type->add_imported_field_name(importD.name); @@ -1011,7 +1003,6 @@ DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) : try { addDataTypeRepo(makeDataTypeRepo(type, *_doc_types), *_doc_types); } catch (...) { - DeleteMapContent(*_doc_types); throw; } } @@ -1029,31 +1020,32 @@ DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) : configureAllRepos(config.documenttype, *_doc_types); } } catch (...) { - DeleteMapContent(*_doc_types); throw; } } DocumentTypeRepo::~DocumentTypeRepo() { - DeleteMapContent(*_doc_types); +} + +DataTypeRepo *DocumentTypeRepo::findRepo(int32_t doc_type_id) const { + return _doc_types->findRepo(doc_type_id); } const DocumentType * DocumentTypeRepo::getDocumentType(int32_t type_id) const noexcept { - const DataTypeRepo *repo = FindPtr(*_doc_types, type_id); - return repo ? repo->doc_type : nullptr; + const DataTypeRepo *repo = findRepo(type_id); + return repo ? repo->doc() : nullptr; } const DocumentType * DocumentTypeRepo::getDocumentType(stringref name) const noexcept { - DocumentTypeMap::const_iterator it = _doc_types->find(DocumentType::createId(name)); - - if (it != _doc_types->end() && it->second->doc_type->getName() == name) { - return it->second->doc_type; + const auto * rp = findRepo(DocumentType::createId(name)); + if (rp && rp->doc()->getName() == name) { + return rp->doc(); } - for (it = _doc_types->begin(); it != _doc_types->end(); ++it) { - if (it->second->doc_type->getName() == name) { - return it->second->doc_type; + for (const auto & [ id, p ] : *_doc_types) { + if (p->doc()->getName() == name) { + return p->doc(); } } return nullptr; @@ -1061,26 +1053,26 @@ DocumentTypeRepo::getDocumentType(stringref name) const noexcept { const DataType * DocumentTypeRepo::getDataType(const DocumentType &doc_type, int32_t id) const { - const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); + const DataTypeRepo *dt_repo = findRepo(doc_type.getId()); return dt_repo ? dt_repo->repo.lookup(id) : nullptr; } const DataType * DocumentTypeRepo::getDataType(const DocumentType &doc_type, stringref name) const { - const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); + const DataTypeRepo *dt_repo = findRepo(doc_type.getId()); return dt_repo ? dt_repo->repo.lookup(name) : nullptr; } const AnnotationType * DocumentTypeRepo::getAnnotationType(const DocumentType &doc_type, int32_t id) const { - const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); + const DataTypeRepo *dt_repo = findRepo(doc_type.getId()); return dt_repo ? dt_repo->annotations.lookup(id) : nullptr; } void DocumentTypeRepo::forEachDocumentType(std::function<void(const DocumentType &)> handler) const { - for (const auto & entry : *_doc_types) { - handler(*entry.second->doc_type); + for (const auto & [ it, rp ] : *_doc_types) { + handler(*rp->doc()); } } diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h index 1a3898a1b65..78f7edd078f 100644 --- a/document/src/vespa/document/repo/documenttyperepo.h +++ b/document/src/vespa/document/repo/documenttyperepo.h @@ -37,9 +37,10 @@ public: void forEachDocumentType(std::function<void(const DocumentType &)> handler) const; const DocumentType *getDefaultDocType() const { return _default; } private: - std::unique_ptr<internal::DocumentTypeMap> _doc_types; const DocumentType * _default; + + DataTypeRepo * findRepo(int32_t doc_type_id) const; }; } // namespace document diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp index bb08f0a5ba1..d8dc58edf20 100644 --- a/document/src/vespa/document/select/branch.cpp +++ b/document/src/vespa/document/select/branch.cpp @@ -12,8 +12,8 @@ And::And(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* na _left(std::move(left)), _right(std::move(right)) { - assert(_left.get()); - assert(_right.get()); + assert(_left); + assert(_right); } @@ -39,11 +39,9 @@ namespace { ResultList traceAndValue(const T& value, std::ostream& out, const Node& leftNode, const Node& rightNode) { - ResultList left = leftNode.contains(value); - out << "And - Left branch returned " << left << ".\n"; - ResultList right = rightNode.contains(value); - out << "And - Right branch returned " << right << ".\n"; - return left && right; + out << "And - Left branch returned " << leftNode.contains(value) << ".\n"; + out << "And - Right branch returned " << rightNode.contains(value) << ".\n"; + return leftNode.contains(value) && rightNode.contains(value); } } @@ -85,11 +83,9 @@ namespace { ResultList traceOrValue(const T& value, std::ostream& out, const Node& leftNode, const Node& rightNode) { - ResultList left = leftNode.contains(value); - out << "Or - Left branch returned " << left << ".\n"; - ResultList right = rightNode.contains(value); - out << "Or - Right branch returned " << right << ".\n"; - return left || right; + out << "Or - Left branch returned " << leftNode.contains(value) << ".\n"; + out << "Or - Right branch returned " << rightNode.contains(value) << ".\n"; + return leftNode.contains(value) || rightNode.contains(value); } } @@ -124,13 +120,11 @@ Not::print(std::ostream& out, bool verbose, const std::string& indent) const namespace { template<typename T> - ResultList traceNotValue(const T& value, std::ostream& out, - const Node& node) + ResultList traceNotValue(const T& value, std::ostream& out, const Node& node) { - ResultList result = node.contains(value); - out << "Not - Child returned " << result + out << "Not - Child returned " << node.contains(value) << ". Returning opposite.\n"; - return !result; + return !node.contains(value); } } diff --git a/document/src/vespa/document/select/compare.cpp b/document/src/vespa/document/select/compare.cpp index 1d792041c4e..36ef3b7cdbe 100644 --- a/document/src/vespa/document/select/compare.cpp +++ b/document/src/vespa/document/select/compare.cpp @@ -54,12 +54,10 @@ namespace { document::BucketId b( static_cast<IntegerValue&>(bVal).getValue()); document::BucketId s( static_cast<IntegerValue&>(nVal).getValue()); - ResultList resultList(Result::get(s.contains(b))); - if (op == FunctionOperator::NE) { - return !resultList; + return ! ResultList(Result::get(s.contains(b))); } - return resultList; + return ResultList(Result::get(s.contains(b))); } else { return ResultList(Result::Invalid); } @@ -87,11 +85,9 @@ namespace { document::BucketId s( static_cast<IntegerValue&>(nVal).getValue()); - ResultList resultList(Result::get(s.contains(b))); - - if (op == FunctionOperator::NE) { - resultList = !resultList; - } + ResultList resultList = (op == FunctionOperator::NE) + ? !ResultList(Result::get(s.contains(b))) + : ResultList(Result::get(s.contains(b))); out << "Checked if " << b.toString() << " is "; if (op == FunctionOperator::NE) { out << "not "; } diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp index e72eb36cbdb..5bb2f510e0d 100644 --- a/document/src/vespa/document/select/resultlist.cpp +++ b/document/src/vespa/document/select/resultlist.cpp @@ -7,7 +7,8 @@ namespace document::select { ResultList::ResultList() = default; - +ResultList::ResultList(ResultList &&) noexcept = default; +ResultList & ResultList::operator = (ResultList &&) noexcept = default; ResultList::~ResultList() = default; ResultList::ResultList(const Result& result) { @@ -15,9 +16,9 @@ ResultList::ResultList(const Result& result) { } void -ResultList::add(const fieldvalue::VariableMap& variables, const Result& result) +ResultList::add(fieldvalue::VariableMap variables, const Result& result) { - _results.push_back(ResultPair(variables, &result)); + _results.emplace_back(std::move(variables), &result); } void @@ -109,7 +110,7 @@ ResultList::operator&&(const ResultList& other) const if (vars.empty()) { resultForNoVariables.set(result.toEnum()); } else { - results.add(vars, result); + results.add(std::move(vars), result); } } } @@ -137,7 +138,7 @@ ResultList::operator||(const ResultList& other) const if (vars.empty()) { resultForNoVariables.set(result.toEnum()); } else { - results.add(vars, result); + results.add(std::move(vars), result); } } } diff --git a/document/src/vespa/document/select/resultlist.h b/document/src/vespa/document/select/resultlist.h index b59a4f1a904..3f810004168 100644 --- a/document/src/vespa/document/select/resultlist.h +++ b/document/src/vespa/document/select/resultlist.h @@ -17,8 +17,10 @@ public: using const_reverse_iterator = Results::const_reverse_iterator; ResultList(); - ResultList(ResultList &&) = default; - ResultList & operator = (ResultList &&) = default; + ResultList(ResultList &&) noexcept; + ResultList & operator = (ResultList &&) noexcept; + ResultList(const ResultList &) = delete; + ResultList & operator = (const ResultList &) = delete; ~ResultList(); /** @@ -26,7 +28,7 @@ public: */ explicit ResultList(const Result& result); - void add(const VariableMap& variables, const Result& result); + void add(VariableMap variables, const Result& result); ResultList operator&&(const ResultList& other) const; ResultList operator||(const ResultList& other) const; diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index e0a2bf84f09..aabd09b2558 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -2,6 +2,7 @@ #include "value.h" #include "operator.h" +#include "variablemap.h" #include <vespa/document/fieldvalue/fieldvalue.h> #include <bitset> #include <ostream> @@ -45,8 +46,7 @@ InvalidValue::operator==(const Value&) const } void -InvalidValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +InvalidValue::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; out << "invalid"; @@ -123,8 +123,7 @@ StringValue::operator==(const Value& value) const } void -StringValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +StringValue::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; out << "\"" << _value << "\""; @@ -156,8 +155,7 @@ IntegerValue::operator==(const Value& value) const } void -IntegerValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +IntegerValue::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; out << _value << 'i'; @@ -189,8 +187,7 @@ FloatValue::operator==(const Value& value) const } void -FloatValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +FloatValue::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; out << _value << 'f'; @@ -312,8 +309,7 @@ ArrayValue::regexTrace(const Value& value, std::ostream& trace) const } void -ArrayValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +ArrayValue::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; out << "<no array representation in language yet>"; diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index cd66bac9635..c770974adfe 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -223,7 +223,7 @@ namespace { class IteratorHandler : public fieldvalue::IteratorHandler { public: IteratorHandler(); - ~IteratorHandler(); + ~IteratorHandler() override; bool hasSingleValue() const; std::unique_ptr<Value> stealSingleValue() &&; std::vector<ArrayValue::VariableValue> stealValues() &&; @@ -263,7 +263,7 @@ IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) { if (!_firstValue && getVariables().empty()) { _firstValue = getInternalValue(fv.getValue()); } else { - _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release())); + _values.emplace_back(std::move(getVariables()), Value::SP(getInternalValue(fv.getValue()).release())); } } diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index 9267ff62e1b..5b47b48dd57 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -74,10 +74,10 @@ FieldPathUpdate::applyTo(Document& doc) const } else { std::unique_ptr<select::Node> whereClause = parseDocumentSelection(_originalWhereClause, *doc.getRepo()); select::ResultList results = whereClause->contains(doc); - for (select::ResultList::const_reverse_iterator i = results.rbegin(); i != results.rend(); ++i) { + for (auto i = results.rbegin(); i != results.rend(); ++i) { LOG(spam, "vars = %s", handler->getVariables().toString().c_str()); if (*i->second == select::Result::True) { - handler->setVariables(i->first); + handler->setVariables(std::move(i->first)); doc.iterateNested(path, *handler); } } diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 2e68d4803cb..1a2afa3621f 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1844,6 +1844,7 @@ "public static com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriorityByName(java.lang.String)", "public void <init>(com.yahoo.document.DocumentTypeManager)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String)", + "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutingPolicyFactory(java.lang.String, com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory)", @@ -3123,7 +3124,8 @@ ], "methods": [ "public abstract boolean encode(com.yahoo.messagebus.Routable, com.yahoo.document.serialization.DocumentSerializer)", - "public abstract com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)" + "public abstract com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", + "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer)" ], "fields": [] }, diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index a9dfe0128e0..7446c681dec 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -20,6 +20,7 @@ import java.util.TreeMap; * * @author Håkon Humberset */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VisitorParameters extends Parameters { private String documentSelection; @@ -47,7 +48,7 @@ public class VisitorParameters extends Parameters { private int maxBucketsPerVisitor = 1; private boolean dynamicallyIncreaseMaxBucketsPerVisitor = false; private float dynamicMaxBucketsIncreaseFactor = 2; - private LoadType loadType = LoadType.DEFAULT; + private LoadType loadType = LoadType.DEFAULT; // TODO: Remove on Vespa 8 private DocumentProtocol.Priority priority = null; private int traceLevel = 0; private ThrottlePolicy throttlePolicy = null; @@ -332,10 +333,18 @@ public class VisitorParameters extends Parameters { throttlePolicy = policy; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public void setLoadType(LoadType loadType) { this.loadType = loadType; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public LoadType getLoadType() { return loadType; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index de09049b49e..c2e6dde7f60 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -53,6 +53,7 @@ public class MessageBusDocumentAccess extends DocumentAccess { * * @param params All parameters for construction. */ + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public MessageBusDocumentAccess(MessageBusParams params) { super(params); this.params = params; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java index 628bca4098f..bb8c3a3b1b1 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java @@ -14,6 +14,7 @@ import static java.util.Objects.requireNonNull; /** * @author Einar M R Rosenvinge */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class MessageBusParams extends DocumentAccessParams { private String routingConfigId = null; @@ -26,12 +27,16 @@ public class MessageBusParams extends DocumentAccessParams { private RPCNetworkParams rpcNetworkParams = new RPCNetworkParams(); private com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(); private SourceSessionParams sourceSessionParams = new SourceSessionParams(); - private LoadTypeSet loadTypes; + private LoadTypeSet loadTypes; // TODO remove on Vespa 8 public MessageBusParams() { this(new LoadTypeSet()); } + /** + * @deprecated load types are deprecated. Use default constructor instead + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public MessageBusParams(LoadTypeSet loadTypes) { this.loadTypes = loadTypes; } @@ -39,7 +44,9 @@ public class MessageBusParams extends DocumentAccessParams { /** * * @return Returns the set of load types accepted by this Vespa installation + * @deprecated load types are deprecated */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public LoadTypeSet getLoadTypes() { return loadTypes; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java index 53c09dcbcb6..133736a8542 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java @@ -4,8 +4,10 @@ package com.yahoo.documentapi.messagebus.loadtypes; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; /** + * @deprecated load types are deprecated * @author thomasg */ +@Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public class LoadType { private int id; private String name; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java index e28d760eddf..a3fbed472f0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java @@ -20,9 +20,15 @@ import java.util.TreeMap; * * For testing, you may want to use the empty constructor and add * load types yourself with addType(). + * + * @deprecated load types are deprecated */ +@Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class LoadTypeSet { + public static final LoadTypeSet EMPTY = new LoadTypeSet(); + class DualMap { Map<String, LoadType> nameMap = new TreeMap<String, LoadType>(); Map<Integer, LoadType> idMap = new HashMap<Integer, LoadType>(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java index d1e3b61f998..21f7c243c6f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java @@ -9,10 +9,11 @@ import com.yahoo.text.Utf8String; /** * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public abstract class DocumentMessage extends Message { private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3; - private LoadType loadType = LoadType.DEFAULT; + private LoadType loadType = LoadType.DEFAULT; // TODO: Remove on Vespa 8 /** * Constructs a new message with no content. @@ -65,10 +66,20 @@ public abstract class DocumentMessage extends Message { this.priority = priority; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public LoadType getLoadType() { return loadType; } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void setLoadType(LoadType loadType) { if (loadType != null) { this.loadType = loadType; @@ -79,7 +90,7 @@ public abstract class DocumentMessage extends Message { @Override public int getApproxSize() { - return 4 + 1; // type + priority + return 4 + 1; // type + priority // TODO update on Vespa 8 to not include deprecated fields } @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index ac946b80429..5db426a5db4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -32,6 +32,7 @@ import static java.util.Objects.requireNonNull; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class DocumentProtocol implements Protocol { private static final Logger log = Logger.getLogger(DocumentProtocol.class.getName()); @@ -246,12 +247,27 @@ public class DocumentProtocol implements Protocol { this(docMan, configId, new LoadTypeSet()); } + public DocumentProtocol(DocumentTypeManager documentTypeManager, + DocumentProtocolPoliciesConfig policiesConfig, + DistributionConfig distributionConfig) { + this(requireNonNull(documentTypeManager), null, new LoadTypeSet(), + requireNonNull(policiesConfig), requireNonNull(distributionConfig)); + } + + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public DocumentProtocol(DocumentTypeManager documentTypeManager, LoadTypeSet loadTypes, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(requireNonNull(documentTypeManager), null, requireNonNull(loadTypes), - requireNonNull(policiesConfig), requireNonNull(distributionConfig)); + requireNonNull(policiesConfig), requireNonNull(distributionConfig)); } + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set) { this(docMan, configId == null ? "client" : configId, set, null, null); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 60c8a613bb5..42172b04b90 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -6,22 +6,17 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import java.util.logging.Level; import com.yahoo.messagebus.Routable; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.objects.Deserializer; -import com.yahoo.vespa.objects.Serializer; import java.util.Map; -import java.util.logging.Logger; import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString; import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString; @@ -86,7 +81,7 @@ public abstract class RoutableFactories60 { DocumentMessage msg = doDecode(in); if (msg != null) { msg.setPriority(DocumentProtocol.getPriority(pri)); - msg.setLoadType(loadTypes.getIdMap().get(loadType)); + msg.setLoadType(loadTypes.getIdMap().get(loadType)); // TODO: ignore on Vespa 8 } return msg; } @@ -136,6 +131,7 @@ public abstract class RoutableFactories60 { return doEncode(reply, out); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { byte pri = in.getByte(null); DocumentReply reply = doDecode(in); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java index d38671fa313..7baa41e5c6a 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java @@ -22,7 +22,7 @@ public interface RoutableFactory { /** * <p>This method encodes the content of the given routable into a byte buffer that can later be decoded using the - * {@link #decode(DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)} method.</p> <p>Return false to signal failure.</p> + * {@link #decode(DocumentDeserializer)} method.</p> <p>Return false to signal failure.</p> * <p>This method is NOT exception safe.</p> * * @param obj The routable to encode. @@ -38,7 +38,15 @@ public interface RoutableFactory { * @param in The buffer to read from. * @param loadTypes The LoadTypeSet to inject into the Routable. * @return The decoded routable. + * @deprecated load types are deprecated. Use method without LoadTypeSet instead */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes); + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + default Routable decode(DocumentDeserializer in) { + return decode(in, LoadTypeSet.EMPTY); + } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 24677a9a322..2360cbe8bc3 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -28,13 +28,21 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 final class RoutableRepository { private static final Logger log = Logger.getLogger(RoutableRepository.class.getName()); private final CopyOnWriteHashMap<Integer, VersionMap> factoryTypes = new CopyOnWriteHashMap<>(); private final CopyOnWriteHashMap<CacheKey, RoutableFactory> cache = new CopyOnWriteHashMap<>(); - private LoadTypeSet loadTypes; + private LoadTypeSet loadTypes; // TODO remove on Vespa 8 + public RoutableRepository() {} + + /** + * @deprecated load types are deprecated. Use default constructor instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public RoutableRepository(LoadTypeSet set) { loadTypes = set; } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java index caed3867d99..b1187d48374 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java @@ -7,7 +7,9 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import org.junit.Test; import static org.junit.Assert.*; +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VisitorParametersTestCase { + // TODO: Remove on Vespa 8 private LoadType loadType = new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3); @SuppressWarnings("removal")// TODO: Vespa 8: remove @@ -21,12 +23,12 @@ public class VisitorParametersTestCase { params.setLibraryParameter("groovy", "dudes"); params.setLibraryParameter("ninja", "turtles"); params.setMaxBucketsPerVisitor(55); - params.setPriority(DocumentProtocol.Priority.HIGHEST); + params.setPriority(DocumentProtocol.Priority.HIGHEST); // TODO: Remove on Vespa 8 params.setRoute("extraterrestrial/highway"); params.setTimeoutMs(1337); params.setMaxPending(111); params.setFieldSet(AllFields.NAME); - params.setLoadType(loadType); + params.setLoadType(loadType); // TODO: Remove on Vespa 8 params.setVisitRemoves(true); params.setVisitInconsistentBuckets(true); params.setTraceLevel(9); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java index 73c343174d4..18269971f88 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java @@ -9,6 +9,8 @@ import static org.junit.Assert.assertEquals; /** * @author thomasg */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 +// TODO Vespa 8: remove test case once load types are gone public class LoadTypesTestCase { @Test diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 5250a6b1db7..deecba96aa6 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -143,6 +143,7 @@ public class Messages60TestCase extends MessagesTestBase { private static final String BUCKET_SPACE = "beartato"; @Override + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void run() { GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); msg.setBucketSpace(BUCKET_SPACE); @@ -151,7 +152,7 @@ public class Messages60TestCase extends MessagesTestBase { for (Language lang : LANGUAGES) { msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("default", msg.getLoadType().getName()); + assertEquals("default", msg.getLoadType().getName()); // TODO: Remove on Vespa 8 assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } @@ -162,9 +163,10 @@ public class Messages60TestCase extends MessagesTestBase { private static final String BUCKET_SPACE = "andrei"; @Override + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void run() { StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); - msg.setLoadType(null); + msg.setLoadType(null); // TODO: Remove on Vespa 8 msg.setBucketSpace(BUCKET_SPACE); assertEquals(BASE_MESSAGE_LENGTH + 27 + serializedLength(BUCKET_SPACE), serialize("StatBucketMessage", msg)); @@ -172,7 +174,7 @@ public class Messages60TestCase extends MessagesTestBase { msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); assertEquals("id.user=123", msg.getDocumentSelection()); - assertEquals("default", msg.getLoadType().getName()); + assertEquals("default", msg.getLoadType().getName()); // TODO: Remove on Vespa 8 assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java index 19f77ee1335..74d06c05383 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java @@ -19,6 +19,7 @@ import static org.junit.Assert.*; /** * @author Simon Thoresen Hult */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public abstract class MessagesTestBase { protected enum Language { @@ -28,7 +29,7 @@ public abstract class MessagesTestBase { protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class); protected final DocumentTypeManager docMan = new DocumentTypeManager(); - protected final LoadTypeSet loadTypes = new LoadTypeSet(); + protected final LoadTypeSet loadTypes = new LoadTypeSet(); // TODO remove on Vespa 8 protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes); public MessagesTestBase() { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java index 0c66c05f35e..ab881e143b7 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java @@ -697,6 +697,7 @@ public class MessageBusVisitorSessionTestCase { } @Test + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void testMessageParameters() { MockSender sender = new MockSender(); MockReceiver receiver = new MockReceiver(); @@ -716,7 +717,7 @@ public class MessageBusVisitorSessionTestCase { params.setTimeoutMs(1337); params.setMaxPending(111); params.setFieldSet(DocIdOnly.NAME); - params.setLoadType(new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3)); + params.setLoadType(new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3)); // TODO: Remove on Vespa 8 params.setVisitRemoves(true); params.setVisitInconsistentBuckets(true); params.setTraceLevel(9); diff --git a/fastos/src/vespa/fastos/app.cpp b/fastos/src/vespa/fastos/app.cpp index 05e885a7d37..8477f79f00d 100644 --- a/fastos/src/vespa/fastos/app.cpp +++ b/fastos/src/vespa/fastos/app.cpp @@ -16,18 +16,6 @@ FastOS_ApplicationInterface::FastOS_ApplicationInterface() : _argc(0), _argv(nullptr) { -#ifdef __linux__ - char * fadvise = getenv("VESPA_FADVISE_OPTIONS"); - if (fadvise != nullptr) { - int fadviseOptions(0); - if (strstr(fadvise, "SEQUENTIAL")) { fadviseOptions |= POSIX_FADV_SEQUENTIAL; } - if (strstr(fadvise, "RANDOM")) { fadviseOptions |= POSIX_FADV_RANDOM; } - if (strstr(fadvise, "WILLNEED")) { fadviseOptions |= POSIX_FADV_WILLNEED; } - if (strstr(fadvise, "DONTNEED")) { fadviseOptions |= POSIX_FADV_DONTNEED; } - if (strstr(fadvise, "NOREUSE")) { fadviseOptions |= POSIX_FADV_NOREUSE; } - FastOS_FileInterface::setDefaultFAdviseOptions(fadviseOptions); - } -#endif } FastOS_ApplicationInterface::~FastOS_ApplicationInterface () = default; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5c76eb274df..d915b18ebdf 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -209,7 +209,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, - List.of("olaa"), "2021-09-13", "2022-04-01", + List.of("olaa"), "2021-09-13", "2022-07-01", "Enable Horizon dashboard", "Takes effect immediately", TENANT_ID, CONSOLE_USER_EMAIL @@ -277,7 +277,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_DATA_HIGHWAY_IN_AWS = defineFeatureFlag( "enable-data-highway-in-aws", false, - List.of("hmusum"), "2022-01-06", "2022-04-06", + List.of("hmusum"), "2022-01-06", "2022-06-01", "Enable Data Highway in AWS", "Takes effect on restart of Docker container", ZONE_ID, APPLICATION_ID); @@ -397,6 +397,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag CONTROLLER_LOCK_SCHEME = defineStringFlag( + "new-controller-lock-scheme", "OLD", + List.of("hmusum"), "2022-04-07", "2022-05-07", + "Lock scheme to application-related controller locks (valid values: OLD, BOTH, NEW)", + "Takes effect immediately", + ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp index 9f251d4d491..953b2e7432e 100644 --- a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp +++ b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/time.h> #include <vespa/fastos/thread.h> +#include <thread> using namespace vespalib; using vespalib::make_string_short::fmt; diff --git a/fnet/src/vespa/fnet/controlpacket.cpp b/fnet/src/vespa/fnet/controlpacket.cpp index 9ff69a76210..9aa03ad5e3a 100644 --- a/fnet/src/vespa/fnet/controlpacket.cpp +++ b/fnet/src/vespa/fnet/controlpacket.cpp @@ -95,7 +95,10 @@ FNET_ControlPacket FNET_ControlPacket::IOCClose(FNET_CMD_IOC_CLOSE); FNET_ControlPacket -FNET_ControlPacket::DetachServerAdapter(FNET_CMD_DETACH_SERVER_ADAPTER); +FNET_ControlPacket::DetachServerAdapterInit(FNET_CMD_DETACH_SERVER_ADAPTER_INIT); + +FNET_ControlPacket +FNET_ControlPacket::DetachServerAdapterFini(FNET_CMD_DETACH_SERVER_ADAPTER_FINI); FNET_ControlPacket FNET_ControlPacket::Execute(FNET_CMD_EXECUTE); diff --git a/fnet/src/vespa/fnet/controlpacket.h b/fnet/src/vespa/fnet/controlpacket.h index 8dd2d034ae6..ad846d37c30 100644 --- a/fnet/src/vespa/fnet/controlpacket.h +++ b/fnet/src/vespa/fnet/controlpacket.h @@ -38,7 +38,8 @@ public: FNET_CMD_IOC_ENABLE_WRITE, FNET_CMD_IOC_HANDSHAKE_ACT, FNET_CMD_IOC_CLOSE, - FNET_CMD_DETACH_SERVER_ADAPTER, + FNET_CMD_DETACH_SERVER_ADAPTER_INIT, + FNET_CMD_DETACH_SERVER_ADAPTER_FINI, FNET_CMD_EXECUTE, FNET_CMD_TIMEOUT, FNET_CMD_BAD_PACKET, @@ -51,7 +52,8 @@ public: static FNET_ControlPacket IOCEnableWrite; static FNET_ControlPacket IOCHandshakeACT; static FNET_ControlPacket IOCClose; - static FNET_ControlPacket DetachServerAdapter; + static FNET_ControlPacket DetachServerAdapterInit; + static FNET_ControlPacket DetachServerAdapterFini; static FNET_ControlPacket Execute; static FNET_ControlPacket Timeout; static FNET_ControlPacket BadPacket; diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index 0a31b9d882b..1681321b239 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -30,12 +30,10 @@ FRT_Supervisor::FRT_Supervisor(FNET_Transport *transport) FRT_Supervisor::~FRT_Supervisor() { + _transport->detach(this); if (_connector != nullptr) { - _connector->Owner()->Close(_connector, /* needref */ false); + _connector->SubRef(); } - _transport->wait_for_pending_resolves(); - _transport->detach(this); - _transport->sync(); } FNET_Scheduler * diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index 0a79f324bb9..1130b6d3e5e 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -125,6 +125,11 @@ TransportConfig::time_tools() const { } // fnet +void +FNET_Transport::wait_for_pending_resolves() { + _async_resolver->wait_for_pending_resolves(); +} + FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) : _async_resolver(cfg.resolver()), _crypto_engine(cfg.crypto()), @@ -158,11 +163,6 @@ FNET_Transport::resolve_async(const vespalib::string &spec, _async_resolver->resolve_async(spec, std::move(result_handler)); } -void -FNET_Transport::wait_for_pending_resolves() { - _async_resolver->wait_for_pending_resolves(); -} - vespalib::CryptoSocket::UP FNET_Transport::create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec) { @@ -221,8 +221,14 @@ void FNET_Transport::detach(FNET_IServerAdapter *server_adapter) { for (const auto &thread: _threads) { - thread->detach(server_adapter); + thread->init_detach(server_adapter); + } + wait_for_pending_resolves(); + sync(); + for (const auto &thread: _threads) { + thread->fini_detach(server_adapter); } + sync(); } FNET_Scheduler * diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 7dbfd80dfe7..d6e4aefb02b 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -113,6 +113,11 @@ private: Threads _threads; const FNET_Config _config; + /** + * Wait for all pending resolve requests. + **/ + void wait_for_pending_resolves(); + public: FNET_Transport(const FNET_Transport &) = delete; FNET_Transport & operator = (const FNET_Transport &) = delete; @@ -160,11 +165,6 @@ public: vespalib::AsyncResolver::ResultHandler::WP result_handler); /** - * Wait for all pending resolve requests. - **/ - void wait_for_pending_resolves(); - - /** * Wrap a plain socket endpoint (client side) in a CryptoSocket. The * implementation will be determined by the CryptoEngine used by * this Transport. @@ -258,11 +258,8 @@ public: * Detach a server adapter from this transport. * * This will close all connectors and connections referencing the - * server adapter. Note that this is an async - * operation. 'wait_for_pending_resolves' should be called before - * this to make sure any in-flight connections are added - * first. 'sync' should be called after this to drain any pending - * call-backs. + * server adapter. Note that this function will also synchronize + * with async address resolving and underlying transport threads. **/ void detach(FNET_IServerAdapter *server_adapter); diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 53363740bae..d46d174c670 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -162,7 +162,7 @@ FNET_TransportThread::SafeDiscardEvent(FNET_ControlPacket *cpacket, void FNET_TransportThread::handle_add_cmd(FNET_IOComponent *ioc) { - if (ioc->handle_add_event()) { + if ((_detaching.count(ioc->server_adapter()) == 0) && ioc->handle_add_event()) { AddComponent(ioc); ioc->_flags._ioc_added = true; ioc->attach_selector(_selector); @@ -186,8 +186,9 @@ FNET_TransportThread::handle_close_cmd(FNET_IOComponent *ioc) void -FNET_TransportThread::handle_detach_server_adapter_cmd(FNET_IServerAdapter *server_adapter) +FNET_TransportThread::handle_detach_server_adapter_init_cmd(FNET_IServerAdapter *server_adapter) { + _detaching.insert(server_adapter); FNET_IOComponent *component = _componentsHead; while (component != nullptr) { FNET_IOComponent *tmp = component; @@ -201,6 +202,12 @@ FNET_TransportThread::handle_detach_server_adapter_cmd(FNET_IServerAdapter *serv } +void +FNET_TransportThread::handle_detach_server_adapter_fini_cmd(FNET_IServerAdapter *server_adapter) +{ + _detaching.erase(server_adapter); +} + extern "C" { static void pipehandler(int) @@ -241,7 +248,8 @@ FNET_TransportThread::FNET_TransportThread(FNET_Transport &owner_in) _pseudo_thread(), _started(false), _shutdown(false), - _finished(false) + _finished(false), + _detaching() { trapsigpipe(); } @@ -348,9 +356,15 @@ FNET_TransportThread::Close(FNET_IOComponent *comp, bool needRef) } void -FNET_TransportThread::detach(FNET_IServerAdapter *server_adapter) +FNET_TransportThread::init_detach(FNET_IServerAdapter *server_adapter) { - PostEvent(&FNET_ControlPacket::DetachServerAdapter, FNET_Context(server_adapter)); + PostEvent(&FNET_ControlPacket::DetachServerAdapterInit, FNET_Context(server_adapter)); +} + +void +FNET_TransportThread::fini_detach(FNET_IServerAdapter *server_adapter) +{ + PostEvent(&FNET_ControlPacket::DetachServerAdapterFini, FNET_Context(server_adapter)); } bool @@ -432,8 +446,13 @@ FNET_TransportThread::handle_wakeup() continue; } - if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER) { - handle_detach_server_adapter_cmd(context._value.SERVER_ADAPTER); + if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER_INIT) { + handle_detach_server_adapter_init_cmd(context._value.SERVER_ADAPTER); + continue; + } + + if (packet->GetCommand() == FNET_ControlPacket::FNET_CMD_DETACH_SERVER_ADAPTER_FINI) { + handle_detach_server_adapter_fini_cmd(context._value.SERVER_ADAPTER); continue; } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index c120894ac9c..b507c5dc31d 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -13,6 +13,7 @@ #include <mutex> #include <condition_variable> #include <chrono> +#include <set> namespace fnet { struct TimeTools; } class FNET_Transport; @@ -51,6 +52,7 @@ private: std::atomic<bool> _started; // event loop started ? std::atomic<bool> _shutdown; // should stop event loop ? std::atomic<bool> _finished; // event loop stopped ? + std::set<FNET_IServerAdapter*> _detaching; // server adapters being detached /** * Add an IOComponent to the list of components. This operation is @@ -143,7 +145,8 @@ private: void handle_add_cmd(FNET_IOComponent *ioc); void handle_close_cmd(FNET_IOComponent *ioc); - void handle_detach_server_adapter_cmd(FNET_IServerAdapter *server_adapter); + void handle_detach_server_adapter_init_cmd(FNET_IServerAdapter *server_adapter); + void handle_detach_server_adapter_fini_cmd(FNET_IServerAdapter *server_adapter); /** * This method is called to initialize the transport thread event @@ -336,16 +339,16 @@ public: void Close(FNET_IOComponent *comp, bool needRef = true); /** - * Detach a server adapter from this transport. - * - * This will close all connectors and connections referencing the - * server adapter. Note that this is an async - * operation. 'wait_for_pending_resolves' (on the owning - * Transport) should be called before this to make sure any - * in-flight connections are added first. 'sync' should be called - * after this to drain any pending call-backs. + * Start the operation of detaching a server adapter from this + * transport. + **/ + void init_detach(FNET_IServerAdapter *server_adapter); + + /** + * Complete the operation of detaching a server adapter from this + * transport. **/ - void detach(FNET_IServerAdapter *server_adapter); + void fini_detach(FNET_IServerAdapter *server_adapter); /** * Post an execution event on the transport event queue. The return diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java index cfa0452ebf9..2aa1d12c491 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java @@ -20,8 +20,11 @@ import java.util.Optional; */ public interface ContainerEngine { - /** Create a new container */ - void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources); + /** + * Create a new container + * @return ContainerData that can be used to write files inside container + */ + ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources); /** Start a created container */ void startContainer(NodeAgentContext context); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index 8a66373c28b..9060261b806 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -41,8 +41,8 @@ public class ContainerOperations { this.containerStatsCollector = new ContainerStatsCollector(cgroup, fileSystem); } - public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) { - containerEngine.createContainer(context, containerData, containerResources); + public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) { + return containerEngine.createContainer(context, containerResources); } public void startContainer(NodeAgentContext context) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java index 26c3d101c69..10c62695f70 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/ContainerData.java @@ -23,14 +23,20 @@ public interface ContainerData { */ void addFile(ContainerPath path, String data, String permissions); - /** Add directory in container at path. */ - void addDirectory(ContainerPath path); + /** + * @param path Container path to create directory at + * @param permissions optional file permissions, see {@link UnixPath#setPermissions(String)} for format. + */ + void addDirectory(ContainerPath path, String... permissions); /** * Symlink to a file in container at path. * @param symlink The path to the symlink inside the container * @param target The path to the target file for the symbolic link inside the container */ - void createSymlink(ContainerPath symlink, Path target); + void addSymlink(ContainerPath symlink, Path target); + + /** Writes all the files, directories and symlinks that were previously added */ + void converge(NodeAgentContext context); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 9da7f1dbdb6..61e777a9576 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -27,9 +27,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; -import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; -import java.nio.file.Path; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -233,10 +231,10 @@ public class NodeAgentImpl implements NodeAgent { } private Container startContainer(NodeAgentContext context) { - ContainerData containerData = createContainerData(context); ContainerResources wantedResources = warmUpDuration(context).isNegative() ? getContainerResources(context) : getContainerResources(context).withUnlimitedCpus(); - containerOperations.createContainer(context, containerData, wantedResources); + ContainerData containerData = containerOperations.createContainer(context, wantedResources); + writeContainerData(context, containerData); containerOperations.startContainer(context); currentRebootGeneration = context.node().wantedRebootGeneration(); @@ -598,29 +596,7 @@ public class NodeAgentImpl implements NodeAgent { } } - protected ContainerData createContainerData(NodeAgentContext context) { - return new ContainerData() { - @Override - public void addFile(ContainerPath path, String data) { - throw new UnsupportedOperationException("addFile not implemented"); - } - - @Override - public void addFile(ContainerPath path, String data, String permissions) { - throw new UnsupportedOperationException("addFile not implemented"); - } - - @Override - public void addDirectory(ContainerPath path) { - throw new UnsupportedOperationException("addDirectory not implemented"); - } - - @Override - public void createSymlink(ContainerPath symlink, Path target) { - throw new UnsupportedOperationException("createSymlink not implemented"); - } - }; - } + protected void writeContainerData(NodeAgentContext context, ContainerData containerData) { } protected List<CredentialsMaintainer> credentialsMaintainers() { return credentialsMaintainers; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java index 61c4e7475a6..175f9a581a4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectory.java @@ -50,21 +50,17 @@ public class MakeDirectory { throw new UncheckedIOException(new NotDirectoryException(path.toString())); } } else { + Optional<String> permissions = attributeSync.getPermissions(); if (createParents) { // We'll skip logging system modification here, as we'll log about the creation // of the directory next. - path.createParents(); + permissions.ifPresentOrElse(path::createParents, path::createParents); } context.recordSystemModification(logger, "Creating directory " + path); systemModified = true; - Optional<String> permissions = attributeSync.getPermissions(); - if (permissions.isPresent()) { - path.createDirectory(permissions.get()); - } else { - path.createDirectory(); - } + permissions.ifPresentOrElse(path::createDirectory, path::createDirectory); } systemModified |= attributeSync.converge(context, attributesCache); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index cc780e277ad..ac5035216e9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -114,9 +114,7 @@ public class UnixPath { public UnixPath writeBytes(byte[] content, String permissions, OpenOption... options) { FileAttribute<?>[] attributes = Optional.ofNullable(permissions) - .map(this::getPosixFilePermissionsFromString) - .map(PosixFilePermissions::asFileAttribute) - .map(attribute -> new FileAttribute<?>[]{attribute}) + .map(this::permissionsAsFileAttributes) .orElseGet(() -> new FileAttribute<?>[0]); Set<OpenOption> optionsSet = options.length == 0 ? DEFAULT_OPEN_OPTIONS : Set.of(options); @@ -202,27 +200,20 @@ public class UnixPath { return ifExists(this::getAttributes); } - public UnixPath createNewFile() { - uncheck(() -> Files.createFile(path)); + public UnixPath createNewFile(String... permissions) { + uncheck(() -> Files.createFile(path, permissionsAsFileAttributes(permissions))); return this; } - public UnixPath createParents() { - uncheck(() -> Files.createDirectories(path.getParent())); + public UnixPath createParents(String... permissions) { + getParent().createDirectories(permissions); return this; } - public UnixPath createDirectory(String permissions) { - Set<PosixFilePermission> set = getPosixFilePermissionsFromString(permissions); - FileAttribute<Set<PosixFilePermission>> attribute = PosixFilePermissions.asFileAttribute(set); - uncheck(() -> Files.createDirectory(path, attribute)); - return this; - } - - /** Create directory unless it already exists, and return this. */ - public UnixPath createDirectory() { + /** Create directory with given permissions, unless it already exists, and return this. */ + public UnixPath createDirectory(String... permissions) { try { - Files.createDirectory(path); + Files.createDirectory(path, permissionsAsFileAttributes(permissions)); } catch (FileAlreadyExistsException ignore) { } catch (IOException e) { throw new UncheckedIOException(e); @@ -230,8 +221,8 @@ public class UnixPath { return this; } - public UnixPath createDirectories() { - uncheck(() -> Files.createDirectories(path)); + public UnixPath createDirectories(String... permissions) { + uncheck(() -> Files.createDirectories(path, permissionsAsFileAttributes(permissions))); return this; } @@ -318,13 +309,19 @@ public class UnixPath { return new UnixPath(link); } - public FileSnapshot getFileSnapshot() { return FileSnapshot.forPath(path); } - @Override public String toString() { return path.toString(); } + private FileAttribute<?>[] permissionsAsFileAttributes(String... permissions) { + if (permissions.length == 0) return new FileAttribute<?>[0]; + if (permissions.length > 1) + throw new IllegalArgumentException("Expected permissions to not be set or be a single string"); + + return new FileAttribute<?>[]{PosixFilePermissions.asFileAttribute(getPosixFilePermissionsFromString(permissions[0]))}; + } + private Set<PosixFilePermission> getPosixFilePermissionsFromString(String permissions) { try { return PosixFilePermissions.fromString(permissions); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java index 25cdff4b726..2d3a4976fe5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngineMock.java @@ -7,8 +7,10 @@ import com.yahoo.vespa.hosted.node.admin.container.image.Image; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; +import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; +import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -66,8 +68,34 @@ public class ContainerEngineMock implements ContainerEngine { } @Override - public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) { + public ContainerData createContainer(NodeAgentContext context, ContainerResources containerResources) { addContainer(createContainer(context, PartialContainer.State.created, containerResources)); + return new ContainerData() { + @Override + public void addFile(ContainerPath path, String data) { + throw new UnsupportedOperationException("addFile not implemented"); + } + + @Override + public void addFile(ContainerPath path, String data, String permissions) { + throw new UnsupportedOperationException("addFile not implemented"); + } + + @Override + public void addDirectory(ContainerPath path, String... permissions) { + throw new UnsupportedOperationException("addDirectory not implemented"); + } + + @Override + public void addSymlink(ContainerPath symlink, Path target) { + throw new UnsupportedOperationException("addSymlink not implemented"); + } + + @Override + public void converge(NodeAgentContext context) { + throw new UnsupportedOperationException("converge not implemented"); + } + }; } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java index a2dbfe0db4b..5649da7c4ea 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.integration; import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.test.file.TestFileSystem; @@ -36,13 +36,13 @@ public class ContainerFailTest { NodeAgentContext context = NodeAgentContextImpl.builder(nodeSpec).fileSystem(TestFileSystem.create()).build(); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any()); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); tester.containerOperations.removeContainer(context, tester.containerOperations.getContainer(context).get()); tester.inOrder(tester.containerOperations).removeContainer(containerMatcher(containerName), any()); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any()); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java index 5bb279323e9..79546d8cf78 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java @@ -48,7 +48,7 @@ public class MultiContainerTest { tester.addChildNodeRepositoryNode(nodeSpec); ContainerName containerName = ContainerName.fromHostname(hostName); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any(), any()); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(containerName), any()); tester.inOrder(tester.containerOperations).resumeNode(containerMatcher(containerName)); tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), any()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java index 4d95991dd29..f3635be1b4b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.integration; import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; import org.junit.Test; @@ -31,7 +31,7 @@ public class RebootTest { tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build()); ContainerName host1 = new ContainerName("host1"); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any(), any()); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); tester.setWantedState(NodeAdminStateUpdater.State.SUSPENDED); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java index 13b3db1c55c..0675626c0e2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.node.admin.integration; import com.yahoo.config.provision.DockerImage; -import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import org.junit.Test; import java.util.List; @@ -29,7 +29,7 @@ public class RestartTest { tester.addChildNodeRepositoryNode(nodeSpec); ContainerName host1 = new ContainerName("host1"); - tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any(), any()); + tester.inOrder(tester.containerOperations).createContainer(containerMatcher(host1), any()); tester.inOrder(tester.nodeRepository).updateNodeAttributes( eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()))); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 28ef19d4065..2eff2c64cec 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -154,7 +154,7 @@ public class NodeAgentImplTest { nodeAgent.stopForHostSuspension(context); nodeAgent.doConverge(context); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); inOrder.verify(containerOperations, times(0)).startServices(eq(context)); // done as part of startContainer inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); @@ -181,7 +181,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer, healthChecker); inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); @@ -310,7 +310,7 @@ public class NodeAgentImplTest { fail("Expected to throw an exception"); } catch (OrchestratorException ignored) { } - verify(containerOperations, never()).createContainer(eq(context), any(), any()); + verify(containerOperations, never()).createContainer(eq(context), any()); verify(containerOperations, never()).startContainer(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); @@ -343,7 +343,7 @@ public class NodeAgentImplTest { // First time we fail to resume because health verification fails verify(orchestrator, times(1)).suspend(eq(hostName)); verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); verify(containerOperations, times(1)).startContainer(eq(context)); verify(orchestrator, never()).resume(eq(hostName)); verify(nodeRepository, never()).updateNodeAttributes(any(), any()); @@ -352,7 +352,7 @@ public class NodeAgentImplTest { // Do not reboot the container again verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); verify(orchestrator, times(1)).resume(eq(hostName)); verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() .withRebootGeneration(wantedRebootGeneration))); @@ -393,7 +393,7 @@ public class NodeAgentImplTest { // Should only be called once, when we initialize verify(containerOperations, times(1)).getContainer(eq(context)); verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, never()).createContainer(eq(context), any(), any()); + verify(containerOperations, never()).createContainer(eq(context), any()); verify(containerOperations, never()).startContainer(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); @@ -458,7 +458,7 @@ public class NodeAgentImplTest { inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready)); - verify(containerOperations, never()).createContainer(eq(context), any(), any()); + verify(containerOperations, never()).createContainer(eq(context), any()); verify(containerOperations, never()).startContainer(eq(context)); verify(containerOperations, never()).suspendNode(eq(context)); verify(containerOperations, times(1)).stopServices(eq(context)); @@ -508,7 +508,7 @@ public class NodeAgentImplTest { nodeAgent.doConverge(context); verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); verify(containerOperations, times(1)).startContainer(eq(context)); } @@ -568,7 +568,7 @@ public class NodeAgentImplTest { } catch (RuntimeException ignored) { } verify(containerOperations, never()).removeContainer(eq(context), any()); - verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + verify(containerOperations, times(1)).createContainer(eq(context), any()); verify(containerOperations, times(1)).startContainer(eq(context)); verify(nodeAgent, never()).resumeNodeIfNeeded(any()); @@ -578,7 +578,7 @@ public class NodeAgentImplTest { nodeAgent.doConverge(context); verify(containerOperations, times(1)).removeContainer(eq(context), any()); - verify(containerOperations, times(2)).createContainer(eq(context), any(), any()); + verify(containerOperations, times(2)).createContainer(eq(context), any()); verify(containerOperations, times(2)).startContainer(eq(context)); verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); } @@ -605,7 +605,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(containerOperations, orchestrator, nodeRepository, aclMaintainer); inOrder.verify(containerOperations, times(1)).pullImageAsyncIfNeeded(any(), eq(dockerImage), any()); - inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any(), any()); + inOrder.verify(containerOperations, times(1)).createContainer(eq(context), any()); inOrder.verify(containerOperations, times(1)).startContainer(eq(context)); inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); @@ -770,10 +770,10 @@ public class NodeAgentImplTest { mockGetContainer(dockerImage, isRunning); doAnswer(invoc -> { NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); - ContainerResources resources = invoc.getArgument(2, ContainerResources.class); + ContainerResources resources = invoc.getArgument(1, ContainerResources.class); mockGetContainer(context.node().wantedDockerImage().get(), resources, true); return null; - }).when(containerOperations).createContainer(any(), any(), any()); + }).when(containerOperations).createContainer(any(), any()); doAnswer(invoc -> { NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index cb82585fe01..fc0da672e6f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -322,7 +322,11 @@ public class Nodes { if (parkOnDeallocationOf(node, agent)) { return park(node.hostname(), false, agent, reason, transaction); } else { - return db.writeTo(Node.State.dirty, List.of(node), agent, Optional.of(reason), transaction).get(0); + Node.State toState = Node.State.dirty; + if (node.state() == Node.State.parked && node.status().wantToDeprovision()) { + throw new IllegalArgumentException("Cannot move " + node + " to " + toState + ": It's being deprovisioned"); + } + return db.writeTo(toState, List.of(node), agent, Optional.of(reason), transaction).get(0); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 379bb2566df..a9abc352d8c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -43,8 +43,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toMap; /** * Client which reads and writes nodes to a curator database. @@ -453,7 +453,8 @@ public class CuratorDatabaseClient { .map(this::readLoadBalancer) .filter(Optional::isPresent) .map(Optional::get) - .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), + .collect(collectingAndThen(toLinkedMap(LoadBalancer::id, + Function.identity()), Collections::unmodifiableMap)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 68d75db8a4c..64cee87d942 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -53,6 +53,7 @@ import static com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner.Beh import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author freva @@ -496,6 +497,12 @@ public class DynamicProvisioningMaintainerTest { tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.system, getClass().getSimpleName()); assertSame("Host moves to parked", Node.State.parked, hostToRemove.get().state()); + // deprovisioning host cannot be unparked + try { + tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.operator, getClass().getSimpleName()); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + // Host and child is removed dynamicProvisioningTester.maintainer.maintain(); allNodes = tester.nodeRepository().nodes().list(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index cc121ba8104..48ee23c7b60 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -334,6 +334,9 @@ public class NodeSerializerTest { " \"hostname\" : \"myHostname\",\n" + " \"ipAddresses\" : [\"127.0.0.1\"],\n" + " \"instance\": {\n" + + " \"tenantId\":\"t\",\n" + + " \"applicationId\":\"a\",\n" + + " \"instanceId\":\"i\",\n" + " \"serviceId\": \"content/myId/0/0/stateful\",\n" + " \"wantedVespaVersion\": \"6.42.2\"\n" + " }\n" + diff --git a/parent/pom.xml b/parent/pom.xml index 5e3ec3d20ea..d90dc7a292f 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -886,6 +886,17 @@ <version>2.3.0</version> </dependency> <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <version>1.1.7</version> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>3.2.5</version> + </dependency> + + <dependency> <groupId>uk.co.datumedge</groupId> <artifactId>hamcrest-json</artifactId> <version>0.2</version> diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java index 79381b8c99e..b9ab1dbe9b6 100644 --- a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/nginx/NginxMetricsReporter.java @@ -58,7 +58,7 @@ public class NginxMetricsReporter extends AbstractComponent implements Runnable @Inject public NginxMetricsReporter(ApplicationIdConfig applicationId, Metric metric, HealthStatus healthStatus, RoutingGenerator routingGenerator) { - this(new ApplicationId(applicationId), metric, healthStatus, FileSystems.getDefault(), interval, routingGenerator::routingTable); + this(ApplicationId.from(applicationId), metric, healthStatus, FileSystems.getDefault(), interval, routingGenerator::routingTable); } NginxMetricsReporter(ApplicationId application, Metric metric, HealthStatus healthStatus, FileSystem fileSystem, Duration interval, diff --git a/screwdriver/release-java-artifacts.sh b/screwdriver/release-java-artifacts.sh index 8030638cf5b..8d80bb45578 100755 --- a/screwdriver/release-java-artifacts.sh +++ b/screwdriver/release-java-artifacts.sh @@ -52,6 +52,8 @@ for MODULE in $(comm -2 -3 \ echo "No javadoc available for module" > $MODULE/src/main/javadoc/README done +# Workaround for broken nexus-staging-maven-plugin instead of swapping JDK +export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.desktop/java.awt.font=ALL-UNNAMED" export VESPA_MAVEN_EXTRA_OPTS="--show-version --batch-mode" ./bootstrap.sh @@ -69,20 +71,12 @@ mvn $COMMON_MAVEN_OPTS --file ./maven-plugins/pom.xml -DskipStagingRepositoryClo # Deploy the rest of the artifacts mvn $COMMON_MAVEN_OPTS --threads 8 -DskipStagingRepositoryClose=true -DstagingRepositoryId=$STG_REPO deploy -# Workaround for nexus-staging-maven-plugin:1.6.12:rc-release not working with maven+jdk17 -SWAP_MAVEN_JAVA_WORKAROUND=false -if rpm -q maven-openjdk17 &> /dev/null; then SWAP_MAVEN_JAVA_WORKAROUND=true; fi -if $SWAP_MAVEN_JAVA_WORKAROUND; then dnf swap -y maven-openjdk17 maven-openjdk11; fi - # Close with checks mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-close -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO # Release if ok mvn $COMMON_MAVEN_OPTS -N org.sonatype.plugins:nexus-staging-maven-plugin:1.6.12:rc-release -DnexusUrl=https://oss.sonatype.org/ -DserverId=ossrh -DstagingRepositoryId=$STG_REPO -# Swap back if we swapped previously -if $SWAP_MAVEN_JAVA_WORKAROUND; then dnf swap -y maven-openjdk11 maven-openjdk17; fi - # Delete the GPG rings rm -rf $SD_SOURCE_DIR/screwdriver/deploy diff --git a/screwdriver/settings-publish.xml b/screwdriver/settings-publish.xml index 5524bf9d7ac..2d6dc2d187c 100644 --- a/screwdriver/settings-publish.xml +++ b/screwdriver/settings-publish.xml @@ -30,7 +30,7 @@ <gpg.publickeyring>pubring.gpg</gpg.publickeyring> <gpg.secretkeyring>secring.gpg</gpg.secretkeyring> <maven.gpg.plugin.version>1.6</maven.gpg.plugin.version> - <nexus.staging.maven.plugin.version>1.6.7</nexus.staging.maven.plugin.version> + <nexus.staging.maven.plugin.version>1.6.12</nexus.staging.maven.plugin.version> </properties> </profile> </profiles> diff --git a/screwdriver/update-vespa-version-in-sample-apps.sh b/screwdriver/update-vespa-version-in-sample-apps.sh index dbcb5d46c3a..ca163de347c 100755 --- a/screwdriver/update-vespa-version-in-sample-apps.sh +++ b/screwdriver/update-vespa-version-in-sample-apps.sh @@ -10,23 +10,25 @@ if [[ $# -ne 1 ]]; then fi readonly VESPA_RELEASE="$1" -readonly TRUE="true" -readonly FALSE="false" - export JAVA_HOME=$(dirname $(dirname $(readlink -f /usr/bin/java))) function is_published { - BUNDLE_PLUGIN_HTTP_CODE=$(curl --write-out %{http_code} --silent --location --output /dev/null https://repo.maven.apache.org/maven2/com/yahoo/vespa/bundle-plugin/${VESPA_RELEASE}/) - if [ $BUNDLE_PLUGIN_HTTP_CODE = "200" ] ; then - echo "$TRUE" + local TMP_MVN_REPO=/tmp/maven-repo + echo $TMP_MVN_REPO + mkdir -p $TMP_MVN_REPO + rm -rf $TMP_MVN_REPO/com/yahoo/vespa + # Because the transfer of artifacts to Maven Central is not atomic we can't just check a simple pom or jar to be available. Because of this we + # check that the publication is complete enough to compile a Java sample app + if mvn -V -B -pl ai.vespa.example:albums -Dmaven.repo.local=$TMP_MVN_REPO -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -DskipTests clean package; then + return 0 else - echo "$FALSE" + return 1 fi } function wait_until_published { cnt=0 - until [[ $(is_published) = "$TRUE" ]]; do + until is_published; do ((cnt+=1)) # Wait max 60 minutes if (( $cnt > 60 )); then @@ -38,8 +40,6 @@ function wait_until_published { done } -wait_until_published - ssh-add -D set +x ssh-add <(echo $SAMPLE_APPS_DEPLOY_KEY | base64 -d) @@ -49,7 +49,9 @@ git clone git@github.com:vespa-engine/sample-apps.git cd sample-apps # Update Vespa version property in pom.xml -mvn versions:set-property -Dproperty=vespa_version -DnewVersion=${VESPA_RELEASE} +mvn -V -B versions:set-property -Dproperty=vespa_version -DnewVersion=${VESPA_RELEASE} + +wait_until_published changes=$(git status --porcelain | wc -l) diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index ab273fd2660..27ecbd08917 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -12,9 +12,11 @@ #include <vespa/config/common/configcontext.h> #include <vespa/fnet/transport.h> #include <vespa/fastos/thread.h> +#include <vespa/fastos/file.h> #include <vespa/fastos/app.h> #include <iostream> #include <thread> +#include <fcntl.h> #include <vespa/log/log.h> LOG_SETUP("proton"); @@ -42,6 +44,7 @@ class App : public FastOS_Application { private: static void setupSignals(); + static void setup_fadvise(); Params parseParams(); void startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport); public: @@ -56,6 +59,23 @@ App::setupSignals() SIG::TERM.hook(); } +void +App::setup_fadvise() +{ +#ifdef __linux__ + char * fadvise = getenv("VESPA_FADVISE_OPTIONS"); + if (fadvise != nullptr) { + int fadviseOptions(0); + if (strstr(fadvise, "SEQUENTIAL")) { fadviseOptions |= POSIX_FADV_SEQUENTIAL; } + if (strstr(fadvise, "RANDOM")) { fadviseOptions |= POSIX_FADV_RANDOM; } + if (strstr(fadvise, "WILLNEED")) { fadviseOptions |= POSIX_FADV_WILLNEED; } + if (strstr(fadvise, "DONTNEED")) { fadviseOptions |= POSIX_FADV_DONTNEED; } + if (strstr(fadvise, "NOREUSE")) { fadviseOptions |= POSIX_FADV_NOREUSE; } + FastOS_FileInterface::setDefaultFAdviseOptions(fadviseOptions); + } +#endif +} + Params App::parseParams() { @@ -247,6 +267,7 @@ App::Main() { try { setupSignals(); + setup_fadvise(); FastOS_ThreadPool threadPool(128_Ki); FNET_Transport transport(buildTransportConfig()); transport.Start(&threadPool); diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 1feb46cfdb7..7ff5eb823ca 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -220,7 +220,11 @@ struct Setup { } bool verify() { generate(); - return vespalib::Process::run(fmt("%s dir:%s", prog, gen_dir.c_str())); + vespalib::Process process(fmt("%s dir:%s", prog, gen_dir.c_str()), true); + for (auto line = process.read_line(); !line.empty(); line = process.read_line()) { + fprintf(stderr, "> %s\n", line.c_str()); + } + return (process.join() == 0); } void verify_valid(std::initializer_list<std::string> features) { for (const std::string &f: features) { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 54561ec6304..4e272e1af4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -24,6 +24,7 @@ #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/vespalib/time/time_box.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/time.h> #include <vespa/config/retriever/configsnapshot.hpp> #include <thread> #include <cassert> @@ -318,6 +319,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot MaintenanceConfigSP newMaintenanceConfig; std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config; std::shared_ptr<const AllocConfig> old_alloc_config; + constexpr vespalib::duration file_resolve_timeout = 60s * 10; if (!_ignoreForwardedConfig) { if (!(_bootstrapConfig->getDocumenttypesConfigSP() && @@ -357,7 +359,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot RankingConstants::Vector constants; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(5*60, 5); + vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const RankingConstantsConfig::Constant &rc : newRankingConstantsConfig->constant) { auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); @@ -373,7 +375,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot RankingExpressions expressions; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(5*60, 5); + vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) { auto desc = fmt("name='%s'", rc.name.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); @@ -389,7 +391,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot OnnxModels::Vector models; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(5*60, 5); + vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const OnnxModelsConfig::Model &rc : newOnnxModelsConfig->model) { auto desc = fmt("name='%s'", rc.name.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp index c6f8ca923e0..79827ce81c0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp @@ -2,6 +2,7 @@ #include "shared_threading_service.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> @@ -21,9 +22,11 @@ SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfi FNET_Transport& transport, storage::spi::BucketExecutor& bucket_executor) : _transport(transport), - _warmup(cfg.warmup_threads(), 128_Ki, CpuUsage::wrap(proton_warmup_executor, CpuUsage::Category::COMPACT)), + _warmup(std::make_unique<vespalib::ThreadStackExecutor>(cfg.warmup_threads(), 128_Ki, + CpuUsage::wrap(proton_warmup_executor, CpuUsage::Category::COMPACT), + cfg.shared_task_limit())), _shared(std::make_shared<vespalib::BlockingThreadStackExecutor>(cfg.shared_threads(), 128_Ki, - cfg.shared_task_limit(), proton_shared_executor)), + cfg.shared_task_limit(), proton_shared_executor)), _field_writer(), _invokeService(std::max(vespalib::adjustTimeoutByDetectedHz(1ms), cfg.field_writer_config().reactionTime())), @@ -51,7 +54,7 @@ SharedThreadingService::~SharedThreadingService() = default; void SharedThreadingService::sync_all_executors() { - _warmup.sync(); + _warmup->sync(); _shared->sync(); if (_field_writer) { _field_writer->sync_all(); diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h index 04e30b0f9b3..ead16441da0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h @@ -3,7 +3,7 @@ #include "i_shared_threading_service.h" #include "shared_threading_service_config.h" -#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/threadexecutor.h> #include <vespa/vespalib/util/syncable.h> #include <vespa/vespalib/util/clock.h> #include <vespa/vespalib/util/invokeserviceimpl.h> @@ -18,7 +18,7 @@ class SharedThreadingService : public ISharedThreadingService { private: using Registration = std::unique_ptr<vespalib::IDestructorCallback>; FNET_Transport & _transport; - vespalib::ThreadStackExecutor _warmup; + std::unique_ptr<vespalib::SyncableThreadExecutor> _warmup; std::shared_ptr<vespalib::SyncableThreadExecutor> _shared; std::unique_ptr<vespalib::ISequencedTaskExecutor> _field_writer; vespalib::InvokeServiceImpl _invokeService; @@ -34,7 +34,7 @@ public: std::shared_ptr<vespalib::Executor> shared_raw() { return _shared; } void sync_all_executors(); - vespalib::ThreadExecutor& warmup() override { return _warmup; } + vespalib::ThreadExecutor& warmup() override { return *_warmup; } vespalib::ThreadExecutor& shared() override { return *_shared; } vespalib::ISequencedTaskExecutor* field_writer() override { return _field_writer.get(); } vespalib::InvokeService & invokeService() override { return _invokeService; } diff --git a/searchlib/src/tests/fef/resolver/resolver_test.cpp b/searchlib/src/tests/fef/resolver/resolver_test.cpp index 77aba1664ad..536507f3776 100644 --- a/searchlib/src/tests/fef/resolver/resolver_test.cpp +++ b/searchlib/src/tests/fef/resolver/resolver_test.cpp @@ -86,4 +86,10 @@ TEST_F("require_that_bad_input_is_handled", Fixture) { EXPECT_TRUE(dynamic_cast<RankingExpressionBlueprint *>(spec[1].blueprint.get()) != nullptr); } +TEST("require that features can be described") { + EXPECT_EQUAL(BlueprintResolver::describe_feature("featureName"), vespalib::string("rank feature 'featureName'")); + EXPECT_EQUAL(BlueprintResolver::describe_feature("rankingExpression(foo)"), vespalib::string("function 'foo'")); + EXPECT_EQUAL(BlueprintResolver::describe_feature("rankingExpression(foo@1234.5678)"), vespalib::string("function 'foo'")); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index b3aa3bd958b..2e562b4a54f 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -65,6 +65,7 @@ vespa_add_library(searchlib_attribute OBJECT loadednumericvalue.cpp loadedvalue.cpp multi_numeric_enum_search_context.cpp + multi_numeric_flag_search_context.cpp multi_numeric_search_context.cpp multi_string_enum_search_context.cpp multi_string_enum_hint_search_context.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index 8765e52d38a..d12c5a7eadc 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -333,7 +333,6 @@ template <typename SC> class FlagAttributeIteratorT : public FlagAttributeIterator { private: - using Attribute = typename SC::Attribute; void doSeek(uint32_t docId) override; protected: @@ -366,7 +365,6 @@ private: using FlagAttributeIteratorT<SC>::setDocId; using FlagAttributeIteratorT<SC>::setAtEnd; using FlagAttributeIteratorT<SC>::isAtEnd; - using Attribute = typename SC::Attribute; using Trinary=vespalib::Trinary; void doSeek(uint32_t docId) override; Trinary is_strict() const override { return Trinary::True; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index 8cde2862645..16b0c0da143 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -307,9 +307,8 @@ void FlagAttributeIteratorStrict<SC>::doSeek(uint32_t docId) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if ((bv != nullptr) && !isAtEnd(docId) && bv->testBit(docId)) { setDocId(docId); return; @@ -318,7 +317,7 @@ FlagAttributeIteratorStrict<SC>::doSeek(uint32_t docId) uint32_t minNextBit(search::endDocId); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr && !isAtEnd(docId)) { uint32_t nextBit = bv->getNextTrueBit(docId); minNextBit = std::min(nextBit, minNextBit); @@ -336,9 +335,8 @@ void FlagAttributeIteratorT<SC>::doSeek(uint32_t docId) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if ((bv != nullptr) && !isAtEnd(docId) && bv->testBit(docId)) { setDocId(docId); return; @@ -351,9 +349,8 @@ void FlagAttributeIteratorT<SC>::or_hits_into(BitVector &result, uint32_t begin_id) { (void) begin_id; const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); for (int i = sc._low; (i <= sc._high); ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr) { result.orWith(*bv); } @@ -364,9 +361,8 @@ template <typename SC> void FlagAttributeIteratorT<SC>::and_hits_into(BitVector &result, uint32_t begin_id) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); if (sc._low == sc._high) { - const BitVector * bv = attr.getBitVector(sc._low); + const BitVector * bv = sc.get_bit_vector(sc._low); if (bv != nullptr) { result.andWith(*bv); } else { @@ -383,18 +379,17 @@ template <typename SC> std::unique_ptr<BitVector> FlagAttributeIteratorT<SC>::get_hits(uint32_t begin_id) { const SC & sc(_concreteSearchCtx); - const Attribute &attr = static_cast<const Attribute &>(sc.attribute()); int i = sc._low; BitVector::UP result; for (;!result && i < sc._high; ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr) { result = BitVector::create(*bv, begin_id, getEndId()); } } for (; i <= sc._high; ++i) { - const BitVector * bv = attr.getBitVector(i); + const BitVector * bv = sc.get_bit_vector(i); if (bv != nullptr) { result->orWith(*bv); } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index f6139c28d65..346c238f0cc 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -2,11 +2,8 @@ #include "flagattribute.h" #include "load_utils.hpp" -#include "attributeiterators.h" #include "multinumericattribute.hpp" - -#include <vespa/searchlib/queryeval/emptysearch.h> -#include <vespa/searchlib/common/bitvectoriterator.h> +#include "multi_numeric_flag_search_context.h" #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.flag_attribute"); @@ -56,7 +53,7 @@ template <typename B> std::unique_ptr<attribute::SearchContext> FlagAttributeT<B>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams &) const { - return std::make_unique<SearchContext>(std::move(qTerm), *this, this->_mvMapping); + return std::make_unique<attribute::MultiNumericFlagSearchContext<typename B::BaseType, typename B::WType>>(std::move(qTerm), *this, this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), _bitVectors); } template <typename B> @@ -232,38 +229,6 @@ FlagAttributeT<B>::removeOldGenerations(vespalib::GenerationHandler::generation_ _bitVectorHolder.trimHoldLists(firstUsed); } -template <typename B> -FlagAttributeT<B>::SearchContext::SearchContext(QueryTermSimple::UP qTerm, const FlagAttributeT<B> & toBeSearched, const MvMapping& mv_mapping) - : BaseSC(std::move(qTerm), toBeSearched, mv_mapping), - _zeroHits(false) -{ -} - -template <typename B> -SearchIterator::UP -FlagAttributeT<B>::SearchContext::createIterator(fef::TermFieldMatchData * matchData, bool strict) -{ - if (this->valid()) { - if (this->_low == this->_high) { - const Attribute & attr(static_cast<const Attribute &>(this->attribute())); - const BitVector * bv(attr.getBitVector(this->_low)); - if (bv != nullptr) { - return BitVectorIterator::create(bv, attr.getCommittedDocIdLimit(), *matchData, strict); - } else { - return std::make_unique<queryeval::EmptySearch>(); - } - } else { - SearchIterator::UP flagIterator( - strict - ? new FlagAttributeIteratorStrict<typename FlagAttributeT<B>::SearchContext>(*this, matchData) - : new FlagAttributeIteratorT<typename FlagAttributeT<B>::SearchContext>(*this, matchData)); - return flagIterator; - } - } else { - return std::make_unique<queryeval::EmptySearch>(); - } -} - template class FlagAttributeT<FlagBaseImpl>; } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index c1ac4e007bf..24bec517eb0 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -7,7 +7,6 @@ namespace search { typedef MultiValueNumericAttribute< IntegerAttributeTemplate<int8_t>, multivalue::Value<int8_t> > FlagBaseImpl; -typedef MultiValueNumericAttribute< IntegerAttributeTemplate<int8_t>, multivalue::Value<int8_t> > HugeFlagBaseImpl; template <typename B> class FlagAttributeT : public B { @@ -15,22 +14,6 @@ public: FlagAttributeT(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); private: typedef AttributeVector::DocId DocId; - using BaseSC = attribute::MultiNumericSearchContext<typename B::BaseType, typename B::WType>; - class SearchContext : public BaseSC { - public: - typedef FlagAttributeT<B> Attribute; - using MvMapping = attribute::MultiValueMapping<typename B::WType>; - SearchContext(std::unique_ptr<QueryTermSimple> qTerm, const FlagAttributeT<B> & toBeSearched, const MvMapping& mv_mapping); - - std::unique_ptr<queryeval::SearchIterator> - createIterator(fef::TermFieldMatchData * matchData, bool strict) override; - - private: - bool _zeroHits; - - template <class SC> friend class FlagAttributeIteratorT; - template <class SC> friend class FlagAttributeIteratorStrict; - }; bool onLoad(vespalib::Executor *executor) override; bool onLoadEnumerated(ReaderBase &attrReader) override; std::unique_ptr<attribute::SearchContext> @@ -50,20 +33,14 @@ private: void resizeBitVectors(uint32_t neededSize); void removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) override; uint32_t getOffset(int8_t value) const { return value + 128; } - BitVector * getBitVector(typename B::BaseType value) const { - return _bitVectors[value + 128]; - } vespalib::GenerationHolder _bitVectorHolder; std::vector<std::shared_ptr<BitVector> > _bitVectorStore; std::vector<BitVector *> _bitVectors; uint32_t _bitVectorSize; - template <class SC> friend class FlagAttributeIteratorT; - template <class SC> friend class FlagAttributeIteratorStrict; }; typedef FlagAttributeT<FlagBaseImpl> FlagAttribute; -typedef FlagAttributeT<HugeFlagBaseImpl> HugeFlagAttribute; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h index 44e7fe9491f..b88a10652a7 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h @@ -4,7 +4,7 @@ #include "numeric_search_context.h" #include "enumstore.h" -#include "multi_value_mapping.h" +#include "multi_value_mapping_read_view.h" namespace search::attribute { @@ -18,8 +18,8 @@ class MultiEnumSearchContext : public BaseSC { protected: using DocId = ISearchContext::DocId; - const MultiValueMapping<M>& _mv_mapping; - const EnumStoreT<T>& _enum_store; + MultiValueMappingReadView<M> _mv_mapping_read_view; + const EnumStoreT<T>& _enum_store; int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { return find(docId, elemId, weight); @@ -29,11 +29,11 @@ protected: return find(docId, elemId); } - MultiEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<T>& enum_store); + MultiEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<T>& enum_store); public: int32_t find(DocId doc, int32_t elemId, int32_t & weight) const { - auto indices(_mv_mapping.get(doc)); + auto indices(_mv_mapping_read_view.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { T v = _enum_store.get_value(indices[i].value_ref().load_acquire()); if (this->match(v)) { @@ -46,7 +46,7 @@ public: } int32_t find(DocId doc, int32_t elemId) const { - auto indices(_mv_mapping.get(doc)); + auto indices(_mv_mapping_read_view.get(doc)); for (uint32_t i(elemId); i < indices.size(); i++) { T v = _enum_store.get_value(indices[i].value_ref().load_acquire()); if (this->match(v)) { diff --git a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp index cc1640a08b9..e7901199e50 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp @@ -9,9 +9,9 @@ namespace search::attribute { template <typename T, typename BaseSC, typename M> -MultiEnumSearchContext<T, BaseSC, M>::MultiEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<T>& enum_store) +MultiEnumSearchContext<T, BaseSC, M>::MultiEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<T>& enum_store) : BaseSC(toBeSearched, std::move(matcher)), - _mv_mapping(mv_mapping), + _mv_mapping_read_view(mv_mapping_read_view), _enum_store(enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.h index b70ce2459ee..fe05afc606f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.h @@ -16,7 +16,7 @@ template <typename T, typename M> class MultiNumericEnumSearchContext : public MultiEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>, M> { public: - MultiNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<T>& enum_store); + MultiNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<T>& enum_store); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.hpp index 9780aa7a507..f4f2c2407fc 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_enum_search_context.hpp @@ -8,8 +8,8 @@ namespace search::attribute { template <typename T, typename M> -MultiNumericEnumSearchContext<T, M>::MultiNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<T>& enum_store) - : MultiEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>, M>(NumericRangeMatcher<T>(*qTerm), toBeSearched, mv_mapping, enum_store) +MultiNumericEnumSearchContext<T, M>::MultiNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<T>& enum_store) + : MultiEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>, M>(NumericRangeMatcher<T>(*qTerm), toBeSearched, mv_mapping_read_view, enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp new file mode 100644 index 00000000000..1c187c5dbd6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "multi_numeric_flag_search_context.h" +#include "attributeiterators.hpp" +#include "attributevector.h" +#include <vespa/searchcommon/attribute/multivalue.h> +#include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchlib/query/query_term_simple.h> + +namespace search::attribute { + +using queryeval::SearchIterator; + +template <typename T, typename M> +MultiNumericFlagSearchContext<T, M>::MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors) + : MultiNumericSearchContext<T, M>(std::move(qTerm), toBeSearched, mv_mapping_read_view), + _bit_vectors(bit_vectors), + _zeroHits(false) +{ +} + +template <typename T, typename M> +std::unique_ptr<SearchIterator> +MultiNumericFlagSearchContext<T, M>::createIterator(fef::TermFieldMatchData* matchData, bool strict) +{ + if (this->valid()) { + if (this->_low == this->_high) { + const AttributeVector & attr = this->attribute(); + const BitVector * bv(get_bit_vector(this->_low)); + if (bv != nullptr) { + return BitVectorIterator::create(bv, attr.getCommittedDocIdLimit(), *matchData, strict); + } else { + return std::make_unique<queryeval::EmptySearch>(); + } + } else { + SearchIterator::UP flagIterator( + strict + ? new FlagAttributeIteratorStrict<MultiNumericFlagSearchContext>(*this, matchData) + : new FlagAttributeIteratorT<MultiNumericFlagSearchContext>(*this, matchData)); + return flagIterator; + } + } else { + return std::make_unique<queryeval::EmptySearch>(); + } +} + +template class MultiNumericFlagSearchContext<int8_t, multivalue::Value<int8_t>>; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h new file mode 100644 index 00000000000..00f7077b2d0 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "multi_numeric_search_context.h" + +namespace search { +class BitVector; +template <class SC> class FlagAttributeIteratorT; +template <class SC> class FlagAttributeIteratorStrict; +} + +namespace search::attribute { + +/* + * MultiNumericFlagSearchContext handles the creation of search iterators for + * a query term on a multi value numeric flag attribute vector. + */ +template <typename T, typename M> +class MultiNumericFlagSearchContext : public MultiNumericSearchContext<T, M> +{ +public: + MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors); + + std::unique_ptr<queryeval::SearchIterator> + createIterator(fef::TermFieldMatchData * matchData, bool strict) override; +private: + vespalib::ConstArrayRef<BitVector *> _bit_vectors; + bool _zeroHits; + const BitVector* get_bit_vector(T value) const { + static_assert(std::is_same_v<T, int8_t>, "Flag attribute search context is only supported for int8_t data type"); + return _bit_vectors[value + 128]; + } + + template <class SC> friend class ::search::FlagAttributeIteratorT; + template <class SC> friend class ::search::FlagAttributeIteratorStrict; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h index 7d43e195d00..3649b542e87 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h @@ -3,7 +3,7 @@ #pragma once #include "numeric_search_context.h" -#include "multi_value_mapping.h" +#include "multi_value_mapping_read_view.h" #include "numeric_range_matcher.h" namespace search::attribute { @@ -17,7 +17,7 @@ class MultiNumericSearchContext : public NumericSearchContext<NumericRangeMatche { private: using DocId = ISearchContext::DocId; - const MultiValueMapping<M>& _mv_mapping; + MultiValueMappingReadView<M> _mv_mapping_read_view; int32_t onFind(DocId docId, int32_t elemId, int32_t& weight) const override final { return find(docId, elemId, weight); @@ -28,9 +28,9 @@ private: } public: - MultiNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping); + MultiNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view); int32_t find(DocId doc, int32_t elemId, int32_t & weight) const { - auto values(_mv_mapping.get(doc)); + auto values(_mv_mapping_read_view.get(doc)); for (uint32_t i(elemId); i < values.size(); i++) { if (this->match(values[i].value())) { weight = values[i].weight(); @@ -42,7 +42,7 @@ public: } int32_t find(DocId doc, int32_t elemId) const { - auto values(_mv_mapping.get(doc)); + auto values(_mv_mapping_read_view.get(doc)); for (uint32_t i(elemId); i < values.size(); i++) { if (this->match(values[i].value())) { return i; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp index 8398c921ec6..15b851215f8 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp @@ -10,9 +10,9 @@ namespace search::attribute { template <typename T, typename M> -MultiNumericSearchContext<T, M>::MultiNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping) +MultiNumericSearchContext<T, M>::MultiNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view) : NumericSearchContext<NumericRangeMatcher<T>>(toBeSearched, *qTerm, false), - _mv_mapping(mv_mapping) + _mv_mapping_read_view(mv_mapping_read_view) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.h index 92650851116..3ae342be61b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.h @@ -17,7 +17,7 @@ class MultiStringEnumHintSearchContext : public MultiStringEnumSearchContext<M>, public EnumHintSearchContext { public: - MultiStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values); + MultiStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values); ~MultiStringEnumHintSearchContext() override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.hpp index a6b0f3f5eb9..fc1f72c940f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_hint_search_context.hpp @@ -6,8 +6,8 @@ namespace search::attribute { template <typename M> -MultiStringEnumHintSearchContext<M>::MultiStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values) - : MultiStringEnumSearchContext<M>(std::move(qTerm), cased, toBeSearched, mv_mapping, enum_store), +MultiStringEnumHintSearchContext<M>::MultiStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values) + : MultiStringEnumSearchContext<M>(std::move(qTerm), cased, toBeSearched, mv_mapping_read_view, enum_store), EnumHintSearchContext(enum_store.get_dictionary(), doc_id_limit, num_values) { diff --git a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.h index a4f05a5c9cc..1787ea0086d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.h @@ -15,7 +15,7 @@ template <typename M> class MultiStringEnumSearchContext : public MultiEnumSearchContext<const char*, StringSearchContext, M> { public: - MultiStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<const char*>& enum_store); + MultiStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<const char*>& enum_store); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.hpp index 02a740b06dc..1d74db04373 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_string_enum_search_context.hpp @@ -9,8 +9,8 @@ namespace search::attribute { template <typename M> -MultiStringEnumSearchContext<M>::MultiStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const MultiValueMapping<M>& mv_mapping, const EnumStoreT<const char*>& enum_store) - : MultiEnumSearchContext<const char*, StringSearchContext, M>(StringMatcher(std::move(qTerm), cased), toBeSearched, mv_mapping, enum_store) +MultiStringEnumSearchContext<M>::MultiStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, const EnumStoreT<const char*>& enum_store) + : MultiEnumSearchContext<const char*, StringSearchContext, M>(StringMatcher(std::move(qTerm), cased), toBeSearched, mv_mapping_read_view, enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index 382f5b02642..b57269d04c6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -3,13 +3,14 @@ #pragma once #include "multi_value_mapping_base.h" +#include "multi_value_mapping_read_view.h" #include <vespa/vespalib/datastore/array_store.h> #include <vespa/vespalib/util/address_space.h> namespace search::attribute { /** - * Class for mapping from from document id to an array of values. + * Class for mapping from document id to an array of values. */ template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> > class MultiValueMapping : public MultiValueMappingBase @@ -17,6 +18,7 @@ class MultiValueMapping : public MultiValueMappingBase public: using MultiValueType = EntryT; using RefType = RefT; + using ReadView = MultiValueMappingReadView<EntryT, RefT>; private: using ArrayRef = vespalib::ArrayRef<EntryT>; using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; @@ -39,6 +41,12 @@ public: // compacting enum store (replacing old enum index with updated enum index) ArrayRef get_writable(uint32_t docId) { return _store.get_writable(_indices[docId].load_relaxed()); } + /* + * Readers holding a generation guard can call make_read_view() to + * get a read view to the multi value mapping. Array bound (read_size) must + * be specified by reader, cf. committed docid limit in attribute vectors. + */ + ReadView make_read_view(size_t read_size) const { return ReadView(_indices.make_read_view(read_size), &_store); } // Pass on hold list management to underlying store void transferHoldLists(generation_t generation) { _store.transferHoldLists(generation); } void trimHoldLists(generation_t firstUsed) { _store.trimHoldLists(firstUsed); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h new file mode 100644 index 00000000000..116e069e8b4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/datastore/atomic_entry_ref.h> +#include <vespa/vespalib/datastore/array_store.h> +#include <vespa/vespalib/util/address_space.h> + +namespace search::attribute { + +/** + * Class for mapping from document id to an array of values as reader. + */ +template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> > +class MultiValueMappingReadView +{ + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using Indices = vespalib::ConstArrayRef<AtomicEntryRef>; + using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; + + Indices _indices; + const ArrayStore* _store; +public: + constexpr MultiValueMappingReadView() + : _indices(), + _store(nullptr) + { + } + MultiValueMappingReadView(Indices indices, const ArrayStore* store) + : _indices(indices), + _store(store) + { + } + vespalib::ConstArrayRef<EntryT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } + bool valid() const noexcept { return _store != nullptr; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 57a7c6a3b14..99963094366 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -171,7 +171,7 @@ MultiValueNumericAttribute<B, M>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams & params) const { (void) params; - return std::make_unique<attribute::MultiNumericSearchContext<T, M>>(std::move(qTerm), *this, this->_mvMapping); + return std::make_unique<attribute::MultiNumericSearchContext<T, M>>(std::move(qTerm), *this, this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index 201bff48be7..c35a2e55ec3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -121,7 +121,8 @@ MultiValueNumericEnumAttribute<B, M>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams & params) const { (void) params; - return std::make_unique<attribute::MultiNumericEnumSearchContext<T, M>>(std::move(qTerm), *this, this->_mvMapping, this->_enumStore); + auto doc_id_limit = this->getCommittedDocIdLimit(); + return std::make_unique<attribute::MultiNumericEnumSearchContext<T, M>>(std::move(qTerm), *this, this->_mvMapping.make_read_view(doc_id_limit), this->_enumStore); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index 688bcaf1825..4854728ca37 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -78,7 +78,8 @@ MultiValueNumericPostingAttribute<B, M>::getSearch(QueryTermSimpleUP qTerm, { using BaseSC = attribute::MultiNumericEnumSearchContext<typename B::BaseClass::BaseType, M>; using SC = attribute::NumericPostingSearchContext<BaseSC, SelfType, int32_t>; - BaseSC base_sc(std::move(qTerm), *this, this->_mvMapping, this->_enumStore); + auto doc_id_limit = this->getCommittedDocIdLimit(); + BaseSC base_sc(std::move(qTerm), *this, this->_mvMapping.make_read_view(doc_id_limit), this->_enumStore); return std::make_unique<SC>(std::move(base_sc), params, *this); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index 212a71dad74..2d60887c23b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -43,7 +43,8 @@ MultiValueStringAttributeT<B, M>::getSearch(QueryTermSimpleUP qTerm, const attribute::SearchContextParams &) const { bool cased = this->get_match_is_cased(); - return std::make_unique<attribute::MultiStringEnumHintSearchContext<M>>(std::move(qTerm), cased, *this, this->_mvMapping, this->_enumStore, this->getCommittedDocIdLimit(), this->getStatus().getNumValues()); + auto doc_id_limit = this->getCommittedDocIdLimit(); + return std::make_unique<attribute::MultiStringEnumHintSearchContext<M>>(std::move(qTerm), cased, *this, this->_mvMapping.make_read_view(doc_id_limit), this->_enumStore, doc_id_limit, this->getStatus().getNumValues()); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 2c2ac48979d..d3cd338cacb 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -93,7 +93,8 @@ MultiValueStringPostingAttributeT<B, T>::getSearch(QueryTermSimpleUP qTerm, using BaseSC = attribute::MultiStringEnumSearchContext<T>; using SC = attribute::StringPostingSearchContext<BaseSC, SelfType, int32_t>; bool cased = this->get_match_is_cased(); - BaseSC base_sc(std::move(qTerm), cased, *this, this->_mvMapping, this->_enumStore); + auto doc_id_limit = this->getCommittedDocIdLimit(); + BaseSC base_sc(std::move(qTerm), cased, *this, this->_mvMapping.make_read_view(doc_id_limit), this->_enumStore); return std::make_unique<SC>(std::move(base_sc), params.useBitVector(), *this); } diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index 91e5a36a7cf..f48be061d15 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -26,7 +26,7 @@ DocumentFieldNode::~DocumentFieldNode() = default; DocumentFieldNode::DocumentFieldNode(const DocumentFieldNode & rhs) : DocumentAccessorNode(rhs), - _fieldPath(rhs._fieldPath), + _fieldPath(), _value(rhs._value), _fieldName(rhs._fieldName), _doc(nullptr) @@ -38,7 +38,7 @@ DocumentFieldNode::operator = (const DocumentFieldNode & rhs) { if (this != &rhs) { DocumentAccessorNode::operator=(rhs); - _fieldPath = rhs._fieldPath; + _fieldPath.clear(); _value = rhs._value; _fieldName = rhs._fieldName; _doc = nullptr; @@ -146,7 +146,7 @@ void DocumentFieldNode::onDocType(const DocumentType & docType) _fieldPath.clear(); docType.buildFieldPath(_fieldPath, _fieldName); if (_fieldPath.empty()) { - throw std::runtime_error(make_string("Field %s could not be loacated in documenttype %s", _fieldName.c_str(), docType.getName().c_str())); + throw std::runtime_error(make_string("Field %s could not be located in documenttype %s", _fieldName.c_str(), docType.getName().c_str())); } } diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h index fd3923bd4a0..e1038f73fa3 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h @@ -32,11 +32,13 @@ public: DECLARE_NBO_SERIALIZE; void visitMembers(vespalib::ObjectVisitor &visitor) const override; DECLARE_EXPRESSIONNODE(DocumentFieldNode); - DocumentFieldNode() : _fieldPath(), _value(), _fieldName(), _doc(NULL) { } - ~DocumentFieldNode(); - DocumentFieldNode(vespalib::stringref name) : _fieldPath(), _value(), _fieldName(name), _doc(NULL) { } + DocumentFieldNode() : _fieldPath(), _value(), _fieldName(), _doc(nullptr) { } + ~DocumentFieldNode() override; + DocumentFieldNode(vespalib::stringref name) : _fieldPath(), _value(), _fieldName(name), _doc(nullptr) { } DocumentFieldNode(const DocumentFieldNode & rhs); DocumentFieldNode & operator = (const DocumentFieldNode & rhs); + DocumentFieldNode(DocumentFieldNode && rhs) noexcept = default; + DocumentFieldNode & operator = (DocumentFieldNode && rhs) noexcept = default; const vespalib::string & getFieldName() const override { return _fieldName; } private: class Handler : public document::fieldvalue::IteratorHandler { diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index 2a12867dd33..731306d1bea 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -28,6 +28,10 @@ constexpr int TRACE_SKIP_POS = 10; using Accept = Blueprint::AcceptInput; +vespalib::string describe(const vespalib::string &feature_name) { + return BlueprintResolver::describe_feature(feature_name); +} + bool is_compatible(bool is_object, Accept accept_type) { return ((accept_type == Accept::ANY) || ((accept_type == Accept::OBJECT) == (is_object))); @@ -122,7 +126,7 @@ struct Compiler : public Blueprint::DependencyHandler { should_trace |= (i < TRACE_SKIP_POS); should_trace |= ((end - pos) < (MAX_TRACE_SIZE - TRACE_SKIP_POS)); if (should_trace) { - trace += fmt(" ... needed by rank feature '%s'\n", pos->parser.featureName().c_str()); + trace += fmt(" ... needed by %s\n", describe(pos->parser.featureName()).c_str()); } else if (i == TRACE_SKIP_POS) { trace += fmt(" (skipped %zu entries)\n", (n - MAX_TRACE_SIZE) + 1); } @@ -135,9 +139,9 @@ struct Compiler : public Blueprint::DependencyHandler { failed_set.insert(feature_name); auto trace = make_trace(skip_self); if (trace.empty()) { - LOG(warning, "invalid rank feature '%s': %s", feature_name.c_str(), reason.c_str()); + LOG(warning, "invalid %s: %s", describe(feature_name).c_str(), reason.c_str()); } else { - LOG(warning, "invalid rank feature '%s': %s\n%s", feature_name.c_str(), reason.c_str(), trace.c_str()); + LOG(warning, "invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str()); } } probe_stack(); @@ -264,6 +268,22 @@ BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory, { } +vespalib::string +BlueprintResolver::describe_feature(const vespalib::string &name) +{ + auto parser = std::make_unique<FeatureNameParser>(name); + if (parser->valid() && + (parser->baseName() == "rankingExpression") && + (parser->parameters().size() == 1) && + parser->output().empty()) + { + auto param = parser->parameters()[0]; + param = param.substr(0, param.find("@")); + return fmt("function '%s'", param.c_str()); + } + return fmt("rank feature '%s'", name.c_str()); +} + void BlueprintResolver::addSeed(vespalib::stringref feature) { diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h index 80320ae780a..3e3b5879518 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h @@ -100,6 +100,12 @@ public: BlueprintResolver(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv); + // Describe a feature based on its name (intended for log messages) + // + // rankingExpression(foo@hash) -> function 'foo' + // feature -> rank feature 'feature' + static vespalib::string describe_feature(const vespalib::string &name); + /** * Add a feature name to the list of seeds. During compilation, * blueprints for all seeds and dependencies will be instantiated diff --git a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp index d61d7faef0f..85d1daffd01 100644 --- a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp +++ b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp @@ -19,8 +19,8 @@ bool verifyFeature(const BlueprintFactory &factory, resolver.addSeed(featureName); bool result = resolver.compile(); if (!result) { - LOG(error, "rank feature verification failed: %s (%s)", - featureName.c_str(), desc.c_str()); + LOG(error, "verification failed: %s (%s)", + BlueprintResolver::describe_feature(featureName).c_str(), desc.c_str()); } return result; } diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp index 54f95e1c543..3f2a1bb1969 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp @@ -21,9 +21,10 @@ void DirectTensorStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) { TensorSP* elem = static_cast<TensorSP*>(buffer) + offset; + const auto& empty = empty_entry(); for (size_t i = 0; i < num_elems; ++i) { clean_ctx.extraBytesCleaned((*elem)->get_memory_usage().allocatedBytes()); - *elem = _emptyEntry; + *elem = empty; ++elem; } } diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h index 776920ab930..1f112f1ea28 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h @@ -24,7 +24,7 @@ private: class TensorBufferType : public vespalib::datastore::BufferType<TensorSP> { private: using ParentType = BufferType<TensorSP>; - using ParentType::_emptyEntry; + using ParentType::empty_entry; using CleanContext = typename ParentType::CleanContext; public: TensorBufferType(); diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp index 2e6d771a870..a668387e5bd 100644 --- a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp @@ -166,9 +166,10 @@ void StreamedValueStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) { TensorEntry::SP* elem = static_cast<TensorEntry::SP*>(buffer) + offset; + const auto& empty = empty_entry(); for (size_t i = 0; i < num_elems; ++i) { clean_ctx.extraBytesCleaned((*elem)->get_memory_usage().allocatedBytes()); - *elem = _emptyEntry; + *elem = empty; ++elem; } } diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h index 7d83e9f3335..29201dc0e61 100644 --- a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h @@ -51,7 +51,7 @@ private: class TensorBufferType : public vespalib::datastore::BufferType<TensorEntry::SP> { private: using ParentType = BufferType<TensorEntry::SP>; - using ParentType::_emptyEntry; + using ParentType::empty_entry; using CleanContext = typename ParentType::CleanContext; public: TensorBufferType() noexcept; diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 77c674ff1e6..ee18384598e 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -16,6 +16,7 @@ #include <iterator> #include <vector> #include <chrono> +#include <climits> #include <thread> using namespace document; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index c465f974a9b..9a5fb2bdd13 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -16,6 +16,7 @@ #include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/messagebus/error.h> +#include <climits> #include <vespa/log/log.h> LOG_SETUP(".documentapiconverter"); diff --git a/testutil/src/main/java/com/yahoo/test/OrderTester.java b/testutil/src/main/java/com/yahoo/test/OrderTester.java index 4acba4ee7fe..cc28ca9f469 100644 --- a/testutil/src/main/java/com/yahoo/test/OrderTester.java +++ b/testutil/src/main/java/com/yahoo/test/OrderTester.java @@ -15,8 +15,8 @@ import java.util.List; * */ -public abstract class OrderTester<T extends Comparable<T>> { - private ArrayList<List<T>> groups = new ArrayList<>(); +public abstract class OrderTester<T extends Comparable<? super T>> { + private final ArrayList<List<T>> groups = new ArrayList<>(); abstract protected void lessTest(T a, T b); abstract protected void greaterTest(T a, T b); @@ -24,7 +24,7 @@ public abstract class OrderTester<T extends Comparable<T>> { @SafeVarargs @SuppressWarnings("varargs") - private final OrderTester<T> addGroup(T... group) { + private OrderTester<T> addGroup(T... group) { groups.add(Arrays.asList(group)); return this; } diff --git a/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java b/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java index 850369fbc2e..e95bc056ba8 100644 --- a/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java +++ b/testutil/src/main/java/com/yahoo/test/TotalOrderTester.java @@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue; * @author Vegard Sjonfjell */ -public class TotalOrderTester<T extends Comparable<T>> extends OrderTester<T> { +public class TotalOrderTester<T extends Comparable<? super T>> extends OrderTester<T> { protected void lessTest(T a, T b) throws AssertionError { assertTrue(a + " must be less than " + b, a.compareTo(b) <= -1); } diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 2530e461354..6fe87541448 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -126,7 +126,21 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-feed-client</artifactId> <version>${project.version}</version> - <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-feed-client-api</artifactId> + <version>${project.version}</version> + </dependency> + + <!-- Jackson dependencies used in this module --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> </dependency> </dependencies> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 7b3e488a5a5..29c572422c3 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -236,12 +236,12 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { + handlerTimeout.toMillis(), MILLISECONDS); - Path requestPath = new Path(request.getUri()); + Path requestPath = new Path(request.getUri(), __ -> { }); // No segment validation here, as document IDs can be anything. for (String path : handlers.keySet()) if (requestPath.matches(path)) { Map<Method, Handler> methods = handlers.get(path); if (methods.containsKey(request.getMethod())) - return methods.get(request.getMethod()).handle(request, new DocumentPath(requestPath), responseHandler); + return methods.get(request.getMethod()).handle(request, new DocumentPath(requestPath, request.getUri().getRawPath()), responseHandler); if (request.getMethod() == OPTIONS) options(methods.keySet(), responseHandler); @@ -1458,10 +1458,12 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private static class DocumentPath { private final Path path; + private final String rawPath; private final Optional<Group> group; - DocumentPath(Path path) { + DocumentPath(Path path, String rawPath) { this.path = requireNonNull(path); + this.rawPath = requireNonNull(rawPath); this.group = Optional.ofNullable(path.get("number")).map(unsignedLongParser::parse).map(Group::of) .or(() -> Optional.ofNullable(path.get("group")).map(Group::of)); } @@ -1470,10 +1472,10 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { return new DocumentId("id:" + requireNonNull(path.get("namespace")) + ":" + requireNonNull(path.get("documentType")) + ":" + group.map(Group::docIdPart).orElse("") + - ":" + requireNonNull(path.getRest())); + ":" + String.join("/", requireNonNull(path.getRest()).segments())); // :'( } - String rawPath() { return path.asString(); } + String rawPath() { return rawPath; } Optional<String> documentType() { return Optional.ofNullable(path.get("documentType")); } Optional<String> namespace() { return Optional.ofNullable(path.get("namespace")); } Optional<Group> group() { return group; } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java index 514669fe0ac..d9b1190aaed 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java @@ -18,7 +18,8 @@ public class MessageBusSessionFactory implements SessionFactory { public MessageBusSessionFactory(MessagePropertyProcessor processor) { this(processor, null, null); } - + + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private MessageBusSessionFactory(MessagePropertyProcessor processor, DocumentmanagerConfig documentmanagerConfig, SlobroksConfig slobroksConfig) { diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index e5da51f0918..84fbe63a576 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -33,10 +33,18 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private String defaultDocprocChain = null; private boolean defaultAbortOnDocumentError = true; private boolean defaultAbortOnSendError = true; - private final LoadTypeSet loadTypes; + private final LoadTypeSet loadTypes; // TODO remove on Vespa 8 private boolean configChanged = false; + public MessagePropertyProcessor(FeederConfig config) { + loadTypes = new LoadTypeSet(); + configure(config); + } + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeConfig instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public MessagePropertyProcessor(FeederConfig config, LoadTypeConfig loadTypeCfg) { loadTypes = new LoadTypeSet(); configure(config, loadTypeCfg); @@ -127,11 +135,19 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib return feederOptions; } + /** + * @deprecated load types are deprecated. configure without LoadTypeConfig instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public synchronized void configure(FeederConfig config, LoadTypeConfig loadTypeConfig) { loadTypes.configure(loadTypeConfig); configure(config); } + /** + * @deprecated load types are deprecated + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 LoadTypeSet getLoadTypes() { return loadTypes; } @@ -175,7 +191,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private boolean abortOnDocumentError; private boolean abortOnFeedError; private boolean createIfNonExistent; - private LoadType loadType; + private LoadType loadType; // TODO remove on Vespa 8 private int traceLevel; PropertySetter(Route route, long timeout, long totalTimeout, DocumentProtocol.Priority priority, LoadType loadType, diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java index cd524c07f73..c2985996bd0 100755 --- a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -4,7 +4,6 @@ package com.yahoo.dummyreceiver; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; @@ -68,7 +67,7 @@ public class DummyReceiver implements MessageHandler { } private void init() { - MessageBusParams params = new MessageBusParams(new LoadTypeSet()); + MessageBusParams params = new MessageBusParams(); params.setRPCNetworkParams(new RPCNetworkParams().setIdentity(new Identity(name))); params.setDocumentManagerConfigId("client"); params.getMessageBusParams().setMaxPendingCount(0); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java index 8f5db4adf97..52f2857c7e5 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -153,7 +153,7 @@ public class Arguments { } } - propertyProcessor = new MessagePropertyProcessor(getFeederConfig(), new LoadTypeConfig(new LoadTypeConfig.Builder())); + propertyProcessor = new MessagePropertyProcessor(getFeederConfig()); } private String getParam(List<String> args, String arg) throws IllegalArgumentException { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java index b9917533a62..2454f5c8627 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -27,18 +27,32 @@ import java.util.Map; * * @author bjorncs */ +@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class DocumentRetriever { private final ClusterList clusterList; private final DocumentAccessFactory documentAccessFactory; private final ClientParameters params; - private final LoadTypeSet loadTypeSet; + private final LoadTypeSet loadTypeSet; // TODO remove on Vespa 8 private MessageBusSyncSession session; private MessageBusDocumentAccess documentAccess; public DocumentRetriever(ClusterList clusterList, DocumentAccessFactory documentAccessFactory, + ClientParameters params) { + this.clusterList = clusterList; + this.documentAccessFactory = documentAccessFactory; + this.loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 + this.params = params; + } + + /** + * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. + */ + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + public DocumentRetriever(ClusterList clusterList, + DocumentAccessFactory documentAccessFactory, LoadTypeSet loadTypeSet, ClientParameters params) { this.clusterList = clusterList; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java index fd2c9e964f7..7596246d16e 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java @@ -38,11 +38,12 @@ public class Main { Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown)); } + @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private static DocumentRetriever createDocumentRetriever(ClientParameters params) { return new DocumentRetriever( new ClusterList("client"), new DocumentAccessFactory(), - new LoadTypeSet(params.configId), + new LoadTypeSet(params.configId), // TODO: Remove on Vespa 8 params ); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index 4d79f2f2e1d..0e64f824b63 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -10,7 +10,6 @@ import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.StaticThrottlePolicy; @@ -36,7 +35,7 @@ import java.util.stream.Collectors; public class VdsVisit { private VdsVisitParameters params; - private MessageBusParams mbparams = new MessageBusParams(new LoadTypeSet()); + private MessageBusParams mbparams = new MessageBusParams(); private VisitorSession session; private final VisitorSessionAccessorFactory sessionAccessorFactory; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index d1fbde7dd42..7dfa3a2cf2e 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -8,7 +8,6 @@ import com.yahoo.documentapi.VisitorDestinationParameters; import com.yahoo.documentapi.VisitorDestinationSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.Identity; @@ -209,7 +208,7 @@ public class VdsVisitTarget { public void run() throws Exception { initShutdownHook(); log.log(Level.FINE, "Starting VdsVisitTarget"); - MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet()); + MessageBusParams mbusParams = new MessageBusParams(); mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress)); if (port > 0) { diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java index 8d7483c2196..30d117ab105 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java @@ -129,7 +129,6 @@ public class DocumentRetrieverTest { return new DocumentRetriever( clusterList, mockedFactory, - new LoadTypeSet(), params); } @@ -145,7 +144,7 @@ public class DocumentRetrieverTest { when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); - LoadTypeSet loadTypeSet = new LoadTypeSet(); + LoadTypeSet loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1); DocumentRetriever documentRetriever = new DocumentRetriever( new ClusterList(), diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 20c7d435964..642fdd5c16f 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -583,6 +583,7 @@ "final" ], "methods": [ + "public static com.yahoo.path.Path from(java.util.List)", "public boolean isChildOf(com.yahoo.path.Path)", "public com.yahoo.path.Path append(java.lang.String)", "public com.yahoo.path.Path append(com.yahoo.path.Path)", @@ -3232,6 +3233,7 @@ "methods": [ "public static boolean isTextCharacter(int)", "public static java.util.OptionalInt validateTextString(java.lang.String)", + "public static boolean isValidTextString(java.lang.String)", "public static boolean isDisplayable(int)", "public static java.lang.String stripInvalidCharacters(java.lang.String)", "public static java.lang.String truncate(java.lang.String, int)", diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index b4898b14b86..ed6ae3678f4 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -36,6 +36,11 @@ <artifactId>aircompressor</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <scope>compile</scope> + </dependency> <!-- provided scope --> <dependency> diff --git a/vespajlib/src/main/java/ai/vespa/validation/Name.java b/vespajlib/src/main/java/ai/vespa/validation/Name.java index b275917dcff..a6ab456c285 100644 --- a/vespajlib/src/main/java/ai/vespa/validation/Name.java +++ b/vespajlib/src/main/java/ai/vespa/validation/Name.java @@ -3,8 +3,6 @@ package ai.vespa.validation; import java.util.regex.Pattern; -import static ai.vespa.validation.Validation.requireMatch; - /** * A name has from 1 to 64 {@link String} characters which may be letters, numbers, * dashes or underscores, and must start with a letter. diff --git a/vespajlib/src/main/java/ai/vespa/validation/Validation.java b/vespajlib/src/main/java/ai/vespa/validation/Validation.java index 816ca931c80..292cb2f0aa5 100644 --- a/vespajlib/src/main/java/ai/vespa/validation/Validation.java +++ b/vespajlib/src/main/java/ai/vespa/validation/Validation.java @@ -37,36 +37,36 @@ public class Validation { /** Requires the value to match the given pattern. */ public static String requireMatch(String value, String description, Pattern pattern) { - return require(pattern.matcher(value).matches(), value, description, "must match '" + pattern + "'"); + return require(pattern.matcher(value).matches(), value, description + " must match '" + pattern + "'"); } /** Requires the value to be non-blank. */ public static String requireNonBlank(String value, String description) { - return require( ! value.isBlank(), value, description, "cannot be blank"); + return require( ! value.isBlank(), value, description + " cannot be blank"); } /** Requires the value to be at least the lower bound. */ public static <T extends Comparable<? super T>> T requireAtLeast(T value, String description, T lower) { - return require(lower.compareTo(value) <= 0, value, description, "must be at least '" + lower + "'"); + return require(lower.compareTo(value) <= 0, value, description + " must be at least '" + lower + "'"); } /** Requires the value to be at most the upper bound. */ public static <T extends Comparable<? super T>> T requireAtMost(T value, String description, T upper) { - return require(upper.compareTo(value) >= 0, value, description, "must be at most '" + upper + "'"); + return require(upper.compareTo(value) >= 0, value, description + " must be at most '" + upper + "'"); } /** Requires the value to be at least the lower bound, and at most the upper bound. */ public static <T extends Comparable<? super T>> T requireInRange(T value, String description, T lower, T upper) { if (lower.compareTo(upper) > 0) throw new IllegalArgumentException("lower bound cannot be greater than upper bound, " + "but got '" + lower + "' > '" + upper + "'"); - return require(lower.compareTo(value) <= 0 && upper.compareTo(value) >= 0, value, description, - "must be at least '" + lower + "' and at most '" + upper + "'"); + return require(lower.compareTo(value) <= 0 && upper.compareTo(value) >= 0, value, + description + " must be at least '" + lower + "' and at most '" + upper + "'"); } /** Returns the argument if the condition is true, otherwise throws. */ - public static <T> T require(boolean condition, T value, String description, String requirement) { + public static <T> T require(boolean condition, T value, String description) { if (condition) return value; - throw new IllegalArgumentException(description + " " + requirement + ", but got: '" + value + "'"); + throw new IllegalArgumentException(description + ", but got: '" + value + "'"); } }
\ No newline at end of file diff --git a/vespajlib/src/main/java/ai/vespa/validation/package-info.java b/vespajlib/src/main/java/ai/vespa/validation/package-info.java new file mode 100644 index 00000000000..edbab3a6fd1 --- /dev/null +++ b/vespajlib/src/main/java/ai/vespa/validation/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.validation; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java new file mode 100644 index 00000000000..e65a645f5be --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/ArchiveStreamReader.java @@ -0,0 +1,216 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import com.yahoo.path.Path; +import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.Objects; +import java.util.OptionalLong; +import java.util.function.Predicate; +import java.util.zip.GZIPInputStream; + +/** + * Helper class for safely reading files from a compressed archive. + * + * @author mpolden + */ +public class ArchiveStreamReader implements AutoCloseable { + + private final ArchiveInputStream archiveInputStream; + private final Options options; + + private long totalRead = 0; + private long entriesRead = 0; + + private ArchiveStreamReader(ArchiveInputStream archiveInputStream, Options options) { + this.archiveInputStream = Objects.requireNonNull(archiveInputStream); + this.options = Objects.requireNonNull(options); + } + + /** Create reader for an inputStream containing a tar.gz file */ + public static ArchiveStreamReader ofTarGzip(InputStream inputStream, Options options) { + return new ArchiveStreamReader(new TarArchiveInputStream(Exceptions.uncheck(() -> new GZIPInputStream(inputStream))), options); + } + + /** Create reader for an inputStream containing a ZIP file */ + public static ArchiveStreamReader ofZip(InputStream inputStream, Options options) { + return new ArchiveStreamReader(new ZipArchiveInputStream(inputStream), options); + } + + /** + * Read the next file in this archive and write it to given outputStream. Returns information about the read archive + * file, or null if there are no more files to read. + */ + public ArchiveFile readNextTo(OutputStream outputStream) { + ArchiveEntry entry; + try { + while ((entry = archiveInputStream.getNextEntry()) != null) { + Path path = Path.fromString(requireNormalized(entry.getName(), options.allowDotSegment)); + if (isSymlink(entry)) throw new IllegalArgumentException("Archive entry " + path + " is a symbolic link, which is unsupported"); + if (entry.isDirectory()) continue; + if (!options.pathPredicate.test(path.toString())) continue; + if (++entriesRead > options.maxEntries) throw new IllegalArgumentException("Attempted to read more entries than entry limit of " + options.maxEntries); + + long size = 0; + byte[] buffer = new byte[2048]; + int read; + while ((read = archiveInputStream.read(buffer)) != -1) { + totalRead += read; + size += read; + if (totalRead > options.maxSize) throw new IllegalArgumentException("Total size of archive exceeds size limit of " + options.maxSize + " bytes"); + if (read > options.maxEntrySize) { + if (!options.truncateEntry) throw new IllegalArgumentException("Size of entry " + path + " exceeded entry size limit of " + options.maxEntrySize + " bytes"); + } else { + outputStream.write(buffer, 0, read); + } + } + return new ArchiveFile(path, crc32(entry), size); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return null; + } + + @Override + public void close() { + Exceptions.uncheck(archiveInputStream::close); + } + + /** Information about a file extracted from a compressed archive */ + public static class ArchiveFile { + + private final Path path; + private final OptionalLong crc32; + private final long size; + + public ArchiveFile(Path name, OptionalLong crc32, long size) { + this.path = Objects.requireNonNull(name); + this.crc32 = Objects.requireNonNull(crc32); + if (crc32.isPresent()) { + requireNonNegative("crc32", crc32.getAsLong()); + } + this.size = requireNonNegative("size", size); + } + + /** The path of this file inside its containing archive */ + public Path path() { + return path; + } + + /** The CRC-32 checksum of this file, if any */ + public OptionalLong crc32() { + return crc32; + } + + /** The decompressed size of this file */ + public long size() { + return size; + } + + } + + /** Get the CRC-32 checksum of given archive entry, if any */ + private static OptionalLong crc32(ArchiveEntry entry) { + long crc32 = -1; + if (entry instanceof ZipArchiveEntry) { + crc32 = ((ZipArchiveEntry) entry).getCrc(); + } + return crc32 > -1 ? OptionalLong.of(crc32) : OptionalLong.empty(); + } + + private static boolean isSymlink(ArchiveEntry entry) { + // Symlinks inside ZIP files are not part of the ZIP spec and are only supported by some implementations, such + // as Info-ZIP. + // + // Commons Compress only has limited support for symlinks as they are only detected when the ZIP file is read + // through org.apache.commons.compress.archivers.zip.ZipFile. This is not the case in this class, because it must + // support reading ZIP files from generic input streams. The check below thus always returns false. + if (entry instanceof ZipArchiveEntry) return ((ZipArchiveEntry) entry).isUnixSymlink(); + if (entry instanceof TarArchiveEntry) return ((TarArchiveEntry) entry).isSymbolicLink(); + throw new IllegalArgumentException("Unsupported archive entry " + entry.getClass().getSimpleName() + ", cannot check for symbolic link"); + } + + private static String requireNormalized(String name, boolean allowDotSegment) { + for (var part : name.split("/")) { + if (part.isEmpty() || (!allowDotSegment && part.equals(".")) || part.equals("..")) { + throw new IllegalArgumentException("Unexpected non-normalized path found in zip content: '" + name + "'"); + } + } + return name; + } + + private static long requireNonNegative(String field, long n) { + if (n < 0) throw new IllegalArgumentException(field + " cannot be negative, got " + n); + return n; + } + + /** Options for reading entries of an archive */ + public static class Options { + + private long maxSize = 8 * (long) Math.pow(1024, 3); // 8 GB + private long maxEntrySize = Long.MAX_VALUE; + private long maxEntries = Long.MAX_VALUE; + private boolean truncateEntry = false; + private boolean allowDotSegment = false; + private Predicate<String> pathPredicate = (path) -> true; + + private Options() {} + + /** Returns the standard set of read options */ + public static Options standard() { + return new Options(); + } + + /** Set the maximum total size of decompressed entries. Default is 8 GB */ + public Options maxSize(long size) { + this.maxSize = requireNonNegative("size", size); + return this; + } + + /** Set the maximum size a decompressed entry. Default is no limit */ + public Options maxEntrySize(long size) { + this.maxEntrySize = requireNonNegative("size", size); + return this; + } + + /** Set the maximum number of entries to decompress. Default is no limit */ + public Options maxEntries(long count) { + this.maxEntries = requireNonNegative("count", count); + return this; + } + + /** + * Set whether to truncate the content of an entry exceeding the configured size limit, instead of throwing. + * Default is to throw. + */ + public Options truncateEntry(boolean truncate) { + this.truncateEntry = truncate; + return this; + } + + /** Set a predicate that an entry path must match in order to be extracted. Default is to extract all entries */ + public Options pathPredicate(Predicate<String> predicate) { + this.pathPredicate = predicate; + return this; + } + + /** Set whether to allow single-dot segments in entry paths. Default is false */ + public Options allowDotSegment(boolean allow) { + this.allowDotSegment = allow; + return this; + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdOutputStream.java index c423f4c76bf..f439ee03ea6 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdOutputStream.java @@ -7,7 +7,7 @@ import java.io.OutputStream; /** * @author bjorncs */ -public class ZstdOuputStream extends OutputStream { +public class ZstdOutputStream extends OutputStream { private final ZstdCompressor compressor = new ZstdCompressor(); @@ -19,13 +19,13 @@ public class ZstdOuputStream extends OutputStream { private int inputPosition = 0; private boolean isClosed = false; - public ZstdOuputStream(OutputStream out, int inputBufferSize) { + public ZstdOutputStream(OutputStream out, int inputBufferSize) { this.out = out; this.inputBuffer = new byte[inputBufferSize]; this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)]; } - public ZstdOuputStream(OutputStream out) { + public ZstdOutputStream(OutputStream out) { this(out, DEFAULT_INPUT_BUFFER_SIZE); } diff --git a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java index 3cb1cdf5242..28c3f21b24c 100644 --- a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java +++ b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java @@ -18,49 +18,70 @@ import com.sun.jna.Platform; * Provides functionality only possible through native C library. */ public class NativeIO { - private final static Logger logger = Logger.getLogger(NativeIO.class.getName()); - private final static String DISABLE_NATIVE_IO = "DISABLE_NATIVE_IO"; + private static final Logger logger = Logger.getLogger(NativeIO.class.getName()); + private static final String DISABLE_NATIVE_IO = "DISABLE_NATIVE_IO"; private static final int POSIX_FADV_DONTNEED = 4; // See /usr/include/linux/fadvise.h - private static boolean initialized = false; - private static boolean disabled = false; - private static Throwable initError = null; - static { - try { - if (Platform.isLinux()) { - disabled = System.getenv().containsKey(DISABLE_NATIVE_IO); - if (!disabled) { - Native.register(Platform.C_LIBRARY_NAME); - initialized = true; + private static final InitResult fdField = new InitResult(); + private static class InitResult { + private final boolean initialized; + private final boolean enabled; + private final Field fdField; + private final Throwable initError; + InitResult() { + boolean initComplete = false; + boolean disabled = true; + Field field = null; + Throwable exception = null; + try { + if (Platform.isLinux()) { + disabled = System.getenv().containsKey(DISABLE_NATIVE_IO); + if (!disabled) { + Native.register(Platform.C_LIBRARY_NAME); + field = getField(FileDescriptor.class, "fd"); + initComplete = true; + } + } else { + exception = new RuntimeException("Platform is unsúpported. Only supported on linux."); } + } catch (Throwable throwable) { + exception = throwable; + } + initialized = initComplete; + enabled = ! disabled; + initError = exception; + fdField = field; + } + private static Field getField(Class<?> clazz, String fieldName) throws NoSuchFieldException { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return field; + } + boolean isInitialized() { return initialized; } + boolean isEnabled() { return enabled; } + Throwable getError() { return initError; } + int getNativeFD(FileDescriptor fd) { + try { + return fdField.getInt(fd); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); } - } catch (Throwable throwable) { - initError = throwable; } } - private static final Field fieldFD = getField(FileDescriptor.class, "fd"); - - private static native int posix_fadvise(int fd, long offset, long len, int flag) throws LastErrorException; public NativeIO() { - if (!initialized) { - if (disabled) { - logger.info("Native IO has been disable explicit via system property " + DISABLE_NATIVE_IO); - } else { + if ( ! fdField.isInitialized()) { + if (fdField.isEnabled()) { logger.warning("Native IO not possible due to " + getError().getMessage()); + } else { + logger.info("Native IO has been disable explicit via system property " + DISABLE_NATIVE_IO); } } } - public boolean valid() { return initialized; } - public Throwable getError() { - if (initError != null) { - return initError; - } else { - return new RuntimeException("Platform is unsúpported. Only supported on linux."); - } - } + public boolean valid() { return fdField.isInitialized(); } + public Throwable getError() { return fdField.getError(); } /** * Will hint the OS that data read so far will not be accessed again and should hence be dropped from the buffer cache. @@ -74,8 +95,8 @@ public class NativeIO { logger.warning("Sync failed while dropping cache: " + e.getMessage()); } } - if (initialized) { - posix_fadvise(getNativeFD(fd), offset, len, POSIX_FADV_DONTNEED); + if (valid()) { + posix_fadvise(fdField.getNativeFD(fd), offset, len, POSIX_FADV_DONTNEED); } } /** @@ -100,24 +121,4 @@ public class NativeIO { } } - - private static Field getField(Class<?> clazz, String fieldName) { - Field field; - try { - field = clazz.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - field.setAccessible(true); - return field; - } - - private static int getNativeFD(FileDescriptor fd) { - try { - return fieldFD.getInt(fd); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - } diff --git a/vespajlib/src/main/java/com/yahoo/path/Path.java b/vespajlib/src/main/java/com/yahoo/path/Path.java index 16370059c8c..737a27c57d8 100644 --- a/vespajlib/src/main/java/com/yahoo/path/Path.java +++ b/vespajlib/src/main/java/com/yahoo/path/Path.java @@ -41,6 +41,11 @@ public final class Path { this.delimiter = delimiter; } + /** Creates a new path with the given segments. */ + public static Path from(List<String> segments) { + return new Path(segments, "/"); + } + /** Returns whether this path is an immediate child of the given path */ public boolean isChildOf(Path parent) { return toString().startsWith(parent.toString()) && this.elements.size() -1 == parent.elements.size(); diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 662100aa8ea..adf91a9b21e 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -48,43 +48,17 @@ public final class Text { // The link above notes that 0x7F-0x84 and 0x86-0x9F are discouraged, but they are still allowed - // see http://www.w3.org/International/questions/qa-controls - if (codepoint < 0x80) return allowedAsciiChars[codepoint]; + return (codepoint < 0x80) + ? allowedAsciiChars[codepoint] + : (codepoint < Character.MIN_SURROGATE) || isTextCharAboveMinSurrogate(codepoint); + } + private static boolean isTextCharAboveMinSurrogate(int codepoint) { + if (codepoint <= Character.MAX_HIGH_SURROGATE) return false; if (codepoint < 0xFDD0) return true; if (codepoint <= 0xFDDF) return false; - if (codepoint < 0x1FFFE) return true; - if (codepoint <= 0x1FFFF) return false; - if (codepoint < 0x2FFFE) return true; - if (codepoint <= 0x2FFFF) return false; - if (codepoint < 0x3FFFE) return true; - if (codepoint <= 0x3FFFF) return false; - if (codepoint < 0x4FFFE) return true; - if (codepoint <= 0x4FFFF) return false; - if (codepoint < 0x5FFFE) return true; - if (codepoint <= 0x5FFFF) return false; - if (codepoint < 0x6FFFE) return true; - if (codepoint <= 0x6FFFF) return false; - if (codepoint < 0x7FFFE) return true; - if (codepoint <= 0x7FFFF) return false; - if (codepoint < 0x8FFFE) return true; - if (codepoint <= 0x8FFFF) return false; - if (codepoint < 0x9FFFE) return true; - if (codepoint <= 0x9FFFF) return false; - if (codepoint < 0xAFFFE) return true; - if (codepoint <= 0xAFFFF) return false; - if (codepoint < 0xBFFFE) return true; - if (codepoint <= 0xBFFFF) return false; - if (codepoint < 0xCFFFE) return true; - if (codepoint <= 0xCFFFF) return false; - if (codepoint < 0xDFFFE) return true; - if (codepoint <= 0xDFFFF) return false; - if (codepoint < 0xEFFFE) return true; - if (codepoint <= 0xEFFFF) return false; - if (codepoint < 0xFFFFE) return true; - if (codepoint <= 0xFFFFF) return false; - if (codepoint < 0x10FFFE) return true; - if (codepoint <= 0x10FFFF) return false; - - return true; + if (codepoint < 0x10000) return true; + if (codepoint >= 0x10FFFE) return false; + return (codepoint & 0xffff) < 0xFFFE; } /** @@ -110,6 +84,27 @@ public final class Text { return OptionalInt.empty(); } + /** + * Validates that the given string value only contains text characters. + */ + public static boolean isValidTextString(String string) { + int length = string.length(); + for (int i = 0; i < length; ) { + int codePoint = string.codePointAt(i); + if (codePoint < 0x80) { + if ( ! allowedAsciiChars[codePoint]) return false; + } else if (codePoint >= Character.MIN_SURROGATE) { + if ( ! isTextCharAboveMinSurrogate(codePoint)) return false; + if ( ! Character.isBmpCodePoint(codePoint)) { + i++; + } + } + i++; + } + return true; + } + + /** Returns whether the given code point is displayable. */ public static boolean isDisplayable(int codePoint) { switch (Character.getType(codePoint)) { diff --git a/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java new file mode 100644 index 00000000000..b7f019282b7 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/compress/ArchiveStreamReaderTest.java @@ -0,0 +1,131 @@ +package com.yahoo.compress; + +import com.yahoo.compress.ArchiveStreamReader.Options; +import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author mpolden + */ +class ArchiveStreamReaderTest { + + @Test + void reading() { + Map<String, String> zipFiles = Map.of("foo", "contents of foo", + "bar", "contents of bar", + "baz", "0".repeat(2049)); + Map<String, String> zipContents = new HashMap<>(zipFiles); + zipContents.put("dir/", ""); // Directories are always ignored + Map<String, String> extracted = readAll(zip(zipContents), Options.standard()); + assertEquals(zipFiles, extracted); + } + + @Test + void entry_size_limit() { + Map<String, String> entries = Map.of("foo.xml", "foobar"); + Options options = Options.standard().pathPredicate("foo.xml"::equals).maxEntrySize(1); + try { + readAll(zip(entries), options); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + entries = Map.of("foo.xml", "foobar", + "foo.jar", "0".repeat(100) // File not extracted and thus not subject to size limit + ); + Map<String, String> extracted = readAll(zip(entries), options.maxEntrySize(10)); + assertEquals(Map.of("foo.xml", "foobar"), extracted); + } + + @Test + void size_limit() { + Map<String, String> entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); + try { + readAll(zip(entries), Options.standard().maxSize(4)); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + } + + @Test + void entry_limit() { + Map<String, String> entries = Map.of("foo.xml", "foo", "bar.xml", "bar"); + try { + readAll(zip(entries), Options.standard().maxEntries(1)); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + } + + @Test + void paths() { + Map<String, Boolean> tests = Map.of( + "../../services.xml", true, + "/../.././services.xml", true, + "./application/././services.xml", true, + "application//services.xml", true, + "artifacts/", false, // empty dir + "services..xml", false, + "application/services.xml", false, + "components/foo-bar-deploy.jar", false, + "services.xml", false + ); + + Options options = Options.standard().maxEntrySize(1024); + tests.forEach((name, expectException) -> { + try { + readAll(zip(Map.of(name, "foo")), options.pathPredicate(name::equals)); + assertFalse(expectException, "Expected exception for '" + name + "'"); + } catch (IllegalArgumentException ignored) { + assertTrue(expectException, "Unexpected exception for '" + name + "'"); + } + }); + } + + private static Map<String, String> readAll(InputStream inputStream, Options options) { + ArchiveStreamReader reader = ArchiveStreamReader.ofZip(inputStream, options); + ArchiveStreamReader.ArchiveFile file; + Map<String, String> entries = new HashMap<>(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + while ((file = reader.readNextTo(baos)) != null) { + entries.put(file.path().toString(), baos.toString(StandardCharsets.UTF_8)); + baos.reset(); + } + return entries; + } + + private static InputStream zip(Map<String, String> entries) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipArchiveOutputStream archiveOutputStream = null; + try { + archiveOutputStream = new ZipArchiveOutputStream(baos); + for (var kv : entries.entrySet()) { + String entryName = kv.getKey(); + String contents = kv.getValue(); + ZipArchiveEntry entry = new ZipArchiveEntry(entryName); + archiveOutputStream.putArchiveEntry(entry); + archiveOutputStream.write(contents.getBytes(StandardCharsets.UTF_8)); + archiveOutputStream.closeArchiveEntry(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + if (archiveOutputStream != null) Exceptions.uncheck(archiveOutputStream::close); + } + return new ByteArrayInputStream(baos.toByteArray()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/compress/ZstdOuputStreamTest.java b/vespajlib/src/test/java/com/yahoo/compress/ZstdOutputStreamTest.java index 5f6140271ad..c766c6e0c19 100644 --- a/vespajlib/src/test/java/com/yahoo/compress/ZstdOuputStreamTest.java +++ b/vespajlib/src/test/java/com/yahoo/compress/ZstdOutputStreamTest.java @@ -12,13 +12,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author bjorncs */ -class ZstdOuputStreamTest { +class ZstdOutputStreamTest { @Test void output_stream_compresses_input() throws IOException { byte[] inputData = "The quick brown fox jumps over the lazy dog".getBytes(); ByteArrayOutputStream arrayOut = new ByteArrayOutputStream(); - try (ZstdOuputStream zstdOut = new ZstdOuputStream(arrayOut, 12)) { + try (ZstdOutputStream zstdOut = new ZstdOutputStream(arrayOut, 12)) { zstdOut.write(inputData[0]); zstdOut.write(inputData, 1, inputData.length - 1); } @@ -37,7 +37,7 @@ class ZstdOuputStreamTest { } byte[] inputData = builder.toString().getBytes(); ByteArrayOutputStream arrayOut = new ByteArrayOutputStream(); - try (ZstdOuputStream zstdOut = new ZstdOuputStream(arrayOut)) { + try (ZstdOutputStream zstdOut = new ZstdOutputStream(arrayOut)) { zstdOut.write(inputData); } int compressedSize = arrayOut.toByteArray().length; diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java index a2cb2158278..033918f0bad 100644 --- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; +import org.junit.Ignore; import org.junit.Test; import java.util.OptionalInt; @@ -11,18 +12,18 @@ import static org.junit.Assert.assertTrue; public class TextTestCase { + private static void validateText(OptionalInt expect, String text) { + assertEquals(expect, Text.validateTextString(text)); + assertEquals(expect.isEmpty(), Text.isValidTextString(text)); + } @Test public void testValidateTextString() { - assertFalse(Text.validateTextString("valid").isPresent()); - assertEquals(OptionalInt.of(1), Text.validateTextString("text\u0001text\u0003")); - assertEquals(OptionalInt.of(0xDFFFF), - Text.validateTextString(new StringBuilder().appendCodePoint(0xDFFFF).toString())); - assertEquals(OptionalInt.of(0xDFFFF), - Text.validateTextString(new StringBuilder("foo").appendCodePoint(0xDFFFF).toString())); - assertEquals(OptionalInt.of(0xDFFFF), - Text.validateTextString(new StringBuilder().appendCodePoint(0xDFFFF).append("foo").toString())); - assertEquals(OptionalInt.of(0xDFFFF), - Text.validateTextString(new StringBuilder("foo").appendCodePoint(0xDFFFF).append("foo").toString())); + validateText(OptionalInt.empty(), "valid"); + validateText(OptionalInt.of(1), "text\u0001text\u0003"); + validateText(OptionalInt.of(0xDFFFF), new StringBuilder().appendCodePoint(0xDFFFF).toString()); + validateText(OptionalInt.of(0xDFFFF), new StringBuilder("foo").appendCodePoint(0xDFFFF).toString()); + validateText(OptionalInt.of(0xDFFFF), new StringBuilder().appendCodePoint(0xDFFFF).append("foo").toString()); + validateText(OptionalInt.of(0xDFFFF), new StringBuilder("foo").appendCodePoint(0xDFFFF).append("foo").toString()); } @Test @@ -44,8 +45,9 @@ public class TextTestCase { @Test public void testThatHighSurrogateRequireLowSurrogate() { - assertEquals(OptionalInt.of(0xD800), Text.validateTextString(new StringBuilder().appendCodePoint(0xD800).toString())); - assertEquals(OptionalInt.of(0xD800), Text.validateTextString(new StringBuilder().appendCodePoint(0xD800).append(0x0000).toString())); + validateText(OptionalInt.of(0xD800), new StringBuilder().appendCodePoint(0xD800).toString()); + validateText(OptionalInt.of(0xD800), new StringBuilder().appendCodePoint(0xD800).append(0x0000).toString()); + validateText(OptionalInt.empty(), new StringBuilder().appendCodePoint(0xD800).appendCodePoint(0xDC00).toString()); } @Test @@ -76,4 +78,48 @@ public class TextTestCase { public void testFormat() { assertEquals("foo 3.14", Text.format("%s %.2f", "foo", 3.1415926536)); } + + private static long benchmarkIsValid(String [] strings, int num) { + long sum = 0; + for (int i=0; i < num; i++) { + if (Text.isValidTextString(strings[i%strings.length])) { + sum++; + } + } + return sum; + } + + private static long benchmarkValidate(String [] strings, int num) { + long sum = 0; + for (int i=0; i < num; i++) { + if (Text.validateTextString(strings[i%strings.length]).isEmpty()) { + sum++; + } + } + return sum; + } + + @Ignore + @Test + public void benchmarkTextValidation() { + String [] strings = new String[100]; + for (int i=0; i < strings.length; i++) { + strings[i] = new StringBuilder("some text ").append(i).append("of mine.").appendCodePoint(0xDFFFC).append("foo").toString(); + } + long sum = benchmarkValidate(strings, 1000000); + System.out.println("Warmup num validate = " + sum); + sum = benchmarkIsValid(strings, 1000000); + System.out.println("Warmup num isValid = " + sum); + + long start = System.nanoTime(); + sum = benchmarkValidate(strings, 100000000); + long diff = System.nanoTime() - start; + System.out.println("Validation num validate = " + sum + ". Took " + diff + "ns"); + + start = System.nanoTime(); + sum = benchmarkIsValid(strings, 100000000); + diff = System.nanoTime() - start; + System.out.println("Validation num isValid = " + sum + ". Took " + diff + "ns"); + + } } diff --git a/vespalib/src/tests/slime/slime_binary_format_test.cpp b/vespalib/src/tests/slime/slime_binary_format_test.cpp index 459826d691e..ba2aacb88b9 100644 --- a/vespalib/src/tests/slime/slime_binary_format_test.cpp +++ b/vespalib/src/tests/slime/slime_binary_format_test.cpp @@ -248,9 +248,9 @@ TEST("testCmprUlong") { for (uint32_t n = 1; n <= MAX_CMPR_SIZE; ++n) { TEST_STATE(vespalib::make_string("n = %d", n).c_str()); uint64_t min = (n == 1) ? 0x00 - : (1L << ((n - 1) * 7)); + : (1ULL << ((n - 1) * 7)); uint64_t max = (n == MAX_CMPR_SIZE) ? 0xffffffffffffffff - : (1L << (n * 7)) - 1; + : (1ULL << (n * 7)) - 1; SimpleBuffer expect_min; SimpleBuffer expect_max; for (uint32_t i = 0; i < n; ++i) { diff --git a/vespalib/src/tests/wakeup/wakeup_bench.cpp b/vespalib/src/tests/wakeup/wakeup_bench.cpp index c39b8899159..dc6ca70a4d1 100644 --- a/vespalib/src/tests/wakeup/wakeup_bench.cpp +++ b/vespalib/src/tests/wakeup/wakeup_bench.cpp @@ -137,6 +137,7 @@ struct UsePipe : State { } }; +#if __cpp_lib_atomic_wait struct UseAtomic : State { void wakeup() { set_wakeup(); @@ -151,6 +152,7 @@ struct UseAtomic : State { // assert(!is_ready()); } }; +#endif #ifdef __linux__ struct UseFutex : State { @@ -284,7 +286,9 @@ TEST(WakeupBench, using_spin_yield) { benchmark<Wakeup<UseSpinYield>>(); } TEST(WakeupBench, using_cond) { benchmark<Wakeup<UseCond>>(); } TEST(WakeupBench, using_cond_nolock) { benchmark<Wakeup<UseCondNolock>>(); } TEST(WakeupBench, using_pipe) { benchmark<Wakeup<UsePipe>>(); } +#if __cpp_lib_atomic_wait TEST(WakeupBench, using_atomic) { benchmark<Wakeup<UseAtomic>>(); } +#endif #ifdef __linux__ TEST(WakeupBench, using_futex) { benchmark<Wakeup<UseFutex>>(); } diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index 444bf641899..d4a5ae42ef8 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -25,7 +25,7 @@ template <typename EntryType> class BTreeNodeBufferType : public datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>> { using ParentType = datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>>; - using ParentType::_emptyEntry; + using ParentType::empty_entry; using ParentType::_arraySize; using ElemCount = typename ParentType::ElemCount; using CleanContext = typename ParentType::CleanContext; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index 7d041646ca5..3394af18b04 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -133,7 +133,7 @@ template <typename EntryType, typename EmptyType = EntryType> class BufferType : public BufferTypeBase { protected: - static EntryType _emptyEntry; + static const EntryType& empty_entry() noexcept; public: BufferType() noexcept : BufferType(1,1,1) {} diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp index ae2659ef65b..72c8f574a70 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp @@ -51,8 +51,9 @@ void BufferType<EntryType, EmptyType>::initializeReservedElements(void *buffer, ElemCount reservedElems) { EntryType *e = static_cast<EntryType *>(buffer); + const auto& empty = empty_entry(); for (size_t j = reservedElems; j != 0; --j) { - new (static_cast<void *>(e)) EntryType(_emptyEntry); + new (static_cast<void *>(e)) EntryType(empty); ++e; } } @@ -62,13 +63,22 @@ void BufferType<EntryType, EmptyType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) { EntryType *e = static_cast<EntryType *>(buffer) + offset; + const auto& empty = empty_entry(); for (size_t j = numElems; j != 0; --j) { - *e = _emptyEntry; + *e = empty; ++e; } } template <typename EntryType, typename EmptyType> -EntryType BufferType<EntryType, EmptyType>::_emptyEntry = EmptyType(); +const EntryType& +BufferType<EntryType, EmptyType>::empty_entry() noexcept +{ + // It's possible for EntryType to wrap e.g. an Alloc instance, which has a transitive + // dependency on globally constructed allocator object(s). To avoid issues with global + // construction order, initialize the sentinel on the first access. + static EntryType empty = EmptyType(); + return empty; +} } diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h index 50d15d4a27c..ccaedaa9b64 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h @@ -20,7 +20,7 @@ class LargeArrayBufferType : public BufferType<Array<EntryT>> using AllocSpec = ArrayStoreConfig::AllocSpec; using ArrayType = Array<EntryT>; using ParentType = BufferType<ArrayType>; - using ParentType::_emptyEntry; + using ParentType::empty_entry; using CleanContext = typename ParentType::CleanContext; std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; public: diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp index aeeef8166c6..3042bbff73f 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp @@ -22,9 +22,10 @@ void LargeArrayBufferType<EntryT>::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) { ArrayType* elem = static_cast<ArrayType*>(buffer) + offset; + const auto& empty = empty_entry(); for (size_t i = 0; i < numElems; ++i) { cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); - *elem = _emptyEntry; + *elem = empty; ++elem; } } diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp index 72178f0391b..24136e544b8 100644 --- a/vespalib/src/vespa/vespalib/util/array.hpp +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -60,7 +60,9 @@ Array<T>::Array(const Array & rhs) : _array(rhs._array.create(rhs.size() * sizeof(T))), _sz(rhs.size()) { - construct(array(0), rhs.array(0), _sz, std::is_trivially_copyable<T>()); + if (_sz > 0) [[likely]] { + construct(array(0), rhs.array(0), _sz, std::is_trivially_copyable<T>()); + } } template <typename T> diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h index db3b39f400d..bc1fc540a6c 100644 --- a/vespalib/src/vespa/vespalib/util/arrayref.h +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -42,7 +42,7 @@ public: ConstArrayRef(const SmallVector<T, N> &v) noexcept : _v(&v[0]), _sz(v.size()) { } ConstArrayRef(const ArrayRef<T> & v) noexcept : _v(&v[0]), _sz(v.size()) { } ConstArrayRef(const Array<T> &v) noexcept : _v(&v[0]), _sz(v.size()) { } - ConstArrayRef() noexcept : _v(nullptr), _sz(0) {} + constexpr ConstArrayRef() noexcept : _v(nullptr), _sz(0) {} const T & operator [] (size_t i) const { return _v[i]; } size_t size() const { return _sz; } bool empty() const { return _sz == 0; } diff --git a/vespalib/src/vespa/vespalib/util/fiddle.h b/vespalib/src/vespa/vespalib/util/fiddle.h index 20a13ff4654..f4d2ac33695 100644 --- a/vespalib/src/vespa/vespalib/util/fiddle.h +++ b/vespalib/src/vespa/vespalib/util/fiddle.h @@ -24,8 +24,8 @@ uint32_t mix(uint32_t prefix, uint32_t suffix, uint32_t prefix_bits) { if (prefix_bits >= 32) { return prefix; } - uint32_t suffix_mask = (1 << (32 - prefix_bits)) - 1; - uint32_t prefix_mask = (0 - 1) - suffix_mask; + uint32_t suffix_mask = (1u << (32u - prefix_bits)) - 1u; + uint32_t prefix_mask = (0u - 1u) - suffix_mask; return (prefix & prefix_mask) | (suffix & suffix_mask); } diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.h b/vespalib/src/vespa/vespalib/util/rcuvector.h index 09957d14aaf..ce48082099c 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.h +++ b/vespalib/src/vespa/vespalib/util/rcuvector.h @@ -4,6 +4,7 @@ #include "alloc.h" #include "array.h" +#include "arrayref.h" #include "generationholder.h" #include "growstrategy.h" #include "memoryusage.h" @@ -148,6 +149,15 @@ public: const T& get_elem_ref(size_t i) const noexcept { return _data[i]; } // Called from writer only + /* + * Readers holding a generation guard can call make_read_view() to + * get a read view to the rcu vector. Array bound (read_size) must + * be specified by reader, cf. committed docid limit in attribute vectors. + */ + ConstArrayRef<T> make_read_view(size_t read_size) const noexcept { + return ConstArrayRef<T>(&acquire_elem_ref(0), read_size); + } + void reset(); void shrink(size_t newSize) __attribute__((noinline)); void replaceVector(ArrayType replacement); diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h index a8bc8b102ec..58e05878f64 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.h +++ b/vespamalloc/src/vespamalloc/malloc/common.h @@ -4,7 +4,9 @@ #include <new> #include <atomic> #include <cassert> +#include <cstdio> #include <vespamalloc/util/osmem.h> +#include <thread> extern "C" void MallocRecurseOnSuspend(bool recurse) __attribute__ ((noinline)); diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.cpp b/vespamalloc/src/vespamalloc/malloc/datasegment.cpp index 28b69717fb5..4bb36eade43 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.cpp +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datasegment.h" +#include <unistd.h> namespace vespamalloc::segment { diff --git a/vespamalloc/src/vespamalloc/malloc/freelist.hpp b/vespamalloc/src/vespamalloc/malloc/freelist.hpp index 0bc812b8ad3..9b18c375804 100644 --- a/vespamalloc/src/vespamalloc/malloc/freelist.hpp +++ b/vespamalloc/src/vespamalloc/malloc/freelist.hpp @@ -2,6 +2,7 @@ #pragma once #include "freelist.h" +#include <climits> namespace vespamalloc::segment { diff --git a/vespamalloc/src/vespamalloc/malloc/memorywatcher.h b/vespamalloc/src/vespamalloc/malloc/memorywatcher.h index 88229aca77a..de72f0f3f2e 100644 --- a/vespamalloc/src/vespamalloc/malloc/memorywatcher.h +++ b/vespamalloc/src/vespamalloc/malloc/memorywatcher.h @@ -7,6 +7,7 @@ #include <sys/stat.h> #include <ctype.h> #include <fcntl.h> +#include <unistd.h> #include <vespamalloc/malloc/malloc.h> #include <vespamalloc/util/callstack.h> diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp index 30e9985dae3..1d4cb5a6b5e 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -2,6 +2,7 @@ #include "mmappool.h" #include "common.h" #include <sys/mman.h> +#include <unistd.h> namespace vespamalloc { diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h index aa8679c8171..6c06b840b5b 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.h +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h @@ -2,6 +2,7 @@ #pragma once #include <atomic> +#include <mutex> #include <unordered_map> namespace vespamalloc { diff --git a/vespamalloc/src/vespamalloc/util/osmem.cpp b/vespamalloc/src/vespamalloc/util/osmem.cpp index c61926c084b..e0cfbc36938 100644 --- a/vespamalloc/src/vespamalloc/util/osmem.cpp +++ b/vespamalloc/src/vespamalloc/util/osmem.cpp @@ -2,6 +2,7 @@ #include "osmem.h" #include <vespamalloc/malloc/common.h> #include <cstdio> +#include <cctype> #include <cassert> #include <cerrno> #include <cstdlib> @@ -10,6 +11,7 @@ #include <sys/statfs.h> #include <sys/mman.h> #include <linux/mman.h> +#include <functional> namespace vespamalloc { diff --git a/vsm/src/tests/docsum/docsum.cpp b/vsm/src/tests/docsum/docsum.cpp index 35553c6f98c..475489d2f5a 100644 --- a/vsm/src/tests/docsum/docsum.cpp +++ b/vsm/src/tests/docsum/docsum.cpp @@ -204,12 +204,12 @@ DocsumTest::testSlimeFieldWriter() { FieldPath path; type.buildFieldPath(path, "a"); - fields.push_back(DocsumFieldSpec::FieldIdentifier(0, path)); + fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path))); } { FieldPath path; type.buildFieldPath(path, "c.e"); - fields.push_back(DocsumFieldSpec::FieldIdentifier(0, path)); + fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path))); } sfw.setInputFields(fields); TEST_DO(assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"c\":{\"e\":\"qux\"}}")); @@ -257,14 +257,14 @@ DocsumTest::requireThatSlimeFieldWriterHandlesMap() { FieldPath path; mapType.buildFieldPath(path, "value.b"); - fields.push_back(DocsumFieldSpec::FieldIdentifier(0, path)); + fields.push_back(DocsumFieldSpec::FieldIdentifier(0, std::move(path))); } sfw.setInputFields(fields); TEST_DO(assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"b\":\"bar\"}}]")); { FieldPath path; mapType.buildFieldPath(path, "{k1}.a"); - fields[0] = DocsumFieldSpec::FieldIdentifier(0, path); + fields[0] = DocsumFieldSpec::FieldIdentifier(0, std::move(path)); } sfw.clear(); sfw.setInputFields(fields); diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.cpp b/vsm/src/vespa/vsm/common/documenttypemapping.cpp index 2379103653e..7886c44b2e0 100644 --- a/vsm/src/vespa/vsm/common/documenttypemapping.cpp +++ b/vsm/src/vespa/vsm/common/documenttypemapping.cpp @@ -45,13 +45,13 @@ bool DocumentTypeMapping::prepareBaseDoc(SharedFieldPathMap & map) const { FieldPathMapMapT::const_iterator found = _fieldMap.find(_defaultDocumentTypeName); if (found != _fieldMap.end()) { - map.reset(new FieldPathMapT(found->second)); + map = std::make_shared<FieldPathMapT>(found->second); LOG(debug, "Found FieldPathMap for default document type '%s' with %zd elements", _defaultDocumentTypeName.c_str(), map->size()); } else { LOG(warning, "No FieldPathMap found for default document type '%s'. Using empty one", _defaultDocumentTypeName.c_str()); - map.reset(new FieldPathMapT()); + map = std::make_shared<FieldPathMapT>(); } return true; } @@ -70,7 +70,7 @@ void DocumentTypeMapping::buildFieldMap( highestFNo++; FieldPathMapT & fieldMap = _fieldMap[typeId]; - fieldMap.assign(highestFNo, FieldPath()); + fieldMap.resize(highestFNo); size_t validCount(0); for (StringFieldIdTMapT::const_iterator it = fieldList.begin(), mt = fieldList.end(); it != mt; it++) { diff --git a/vsm/src/vespa/vsm/common/storagedocument.h b/vsm/src/vespa/vsm/common/storagedocument.h index 46a3e2f3251..a7f21cb052f 100644 --- a/vsm/src/vespa/vsm/common/storagedocument.h +++ b/vsm/src/vespa/vsm/common/storagedocument.h @@ -17,7 +17,7 @@ public: class SubDocument { public: - SubDocument() : _fieldValue(NULL) {} + SubDocument() : _fieldValue(nullptr) {} SubDocument(document::FieldValue *fv, document::FieldValue::PathRange nested) : _fieldValue(fv), _range(nested) @@ -43,7 +43,7 @@ public: ~StorageDocument(); const document::Document &docDoc() const { return *_doc; } - bool valid() const { return _doc.get() != NULL; } + bool valid() const { return _doc.get() != nullptr; } const SubDocument &getComplexField(FieldIdT fId) const; const document::FieldValue *getField(FieldIdT fId) const override; bool setField(FieldIdT fId, document::FieldValue::UP fv) override ; diff --git a/vsm/src/vespa/vsm/vsm/docsumfieldspec.cpp b/vsm/src/vespa/vsm/vsm/docsumfieldspec.cpp index ba8310a9879..936aaaa2091 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfieldspec.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfieldspec.cpp @@ -10,9 +10,12 @@ DocsumFieldSpec::FieldIdentifier::FieldIdentifier() : DocsumFieldSpec::FieldIdentifier::FieldIdentifier(FieldIdT id, FieldPath path) : _id(id), - _path(path) + _path(std::move(path)) { } +DocsumFieldSpec::FieldIdentifier::FieldIdentifier(FieldIdentifier &&) noexcept = default; +DocsumFieldSpec::FieldIdentifier & DocsumFieldSpec::FieldIdentifier::operator=(FieldIdentifier &&) noexcept = default; +DocsumFieldSpec::FieldIdentifier::~FieldIdentifier() = default; DocsumFieldSpec::DocsumFieldSpec() : _resultType(search::docsummary::RES_INT), diff --git a/vsm/src/vespa/vsm/vsm/docsumfieldspec.h b/vsm/src/vespa/vsm/vsm/docsumfieldspec.h index 5acef140468..db6ee9fa223 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfieldspec.h +++ b/vsm/src/vespa/vsm/vsm/docsumfieldspec.h @@ -24,6 +24,11 @@ public: public: FieldIdentifier(); FieldIdentifier(FieldIdT id, FieldPath path); + FieldIdentifier(FieldIdentifier &&) noexcept; + FieldIdentifier & operator=(FieldIdentifier &&) noexcept; + FieldIdentifier(const FieldIdentifier &) = delete; + FieldIdentifier & operator=(const FieldIdentifier &) = delete; + ~FieldIdentifier(); FieldIdT getId() const { return _id; } const FieldPath & getPath() const { return _path; } }; @@ -58,7 +63,7 @@ public: } const FieldIdentifier & getOutputField() const { return _outputField; } - void setOutputField(const FieldIdentifier & outputField) { _outputField = outputField; } + void setOutputField(FieldIdentifier outputField) { _outputField = std::move(outputField); } const FieldIdentifierVector & getInputFields() const { return _inputFields; } FieldIdentifierVector & getInputFields() { return _inputFields; } }; diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index bd16c687fc7..70759feb41c 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -118,6 +118,18 @@ public: namespace vsm { +FieldPath +copyPathButFirst(const FieldPath & rhs) { + // skip the element that correspond to the start field value + FieldPath path; + if ( ! rhs.empty()) { + for (auto it = rhs.begin() + 1; it != rhs.end(); ++it) { + path.push_back(std::make_unique<document::FieldPathEntry>(**it)); + } + } + return path; +} + void DocsumFilter::prepareFieldSpec(DocsumFieldSpec & spec, const DocsumTools::FieldSpec & toolsSpec, const FieldMap & fieldMap, const FieldPathMapT & fieldPathMap) @@ -128,14 +140,7 @@ DocsumFilter::prepareFieldSpec(DocsumFieldSpec & spec, const DocsumTools::FieldS FieldIdT field = fieldMap.fieldNo(name); if (field != FieldMap::npos) { if (field < fieldPathMap.size()) { - if (!fieldPathMap[field].empty()) { - // skip the element that correspond to the start field value - spec.setOutputField(DocsumFieldSpec::FieldIdentifier - (field, FieldPath(fieldPathMap[field].begin() + 1, - fieldPathMap[field].end()))); - } else { - spec.setOutputField(DocsumFieldSpec::FieldIdentifier(field, FieldPath())); - } + spec.setOutputField(DocsumFieldSpec::FieldIdentifier(field, copyPathButFirst(fieldPathMap[field]))); } else { LOG(warning, "Could not find a field path for field '%s' with id '%d'", name.c_str(), field); spec.setOutputField(DocsumFieldSpec::FieldIdentifier(field, FieldPath())); @@ -152,18 +157,7 @@ DocsumFilter::prepareFieldSpec(DocsumFieldSpec & spec, const DocsumTools::FieldS if (field != FieldMap::npos) { if (field < fieldPathMap.size()) { LOG(debug, "field %u < map size %zu", field, fieldPathMap.size()); - if (!fieldPathMap[field].empty()) { - FieldPath relPath(fieldPathMap[field].begin() + 1, - fieldPathMap[field].end()); - LOG(debug, "map[%u] -> %zu elements", field, fieldPathMap[field].end() - fieldPathMap[field].begin()); - // skip the element that correspond to the start field value - spec.getInputFields().push_back(DocsumFieldSpec::FieldIdentifier - (field, FieldPath(fieldPathMap[field].begin() + 1, - fieldPathMap[field].end()))); - } else { - LOG(debug, "map[%u] empty", field); - spec.getInputFields().push_back(DocsumFieldSpec::FieldIdentifier(field, FieldPath())); - } + spec.getInputFields().push_back(DocsumFieldSpec::FieldIdentifier(field, copyPathButFirst(fieldPathMap[field]))); } else { LOG(warning, "Could not find a field path for field '%s' with id '%d'", name.c_str(), field); spec.getInputFields().push_back(DocsumFieldSpec::FieldIdentifier(field, FieldPath())); diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index fd02410d03f..86a60702c26 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -86,13 +86,12 @@ </exclusions> </dependency> <!-- snappy-java and metrics-core are included here - to be able to work with ZooKeeper 3.6.2 due to + to be able to work with ZooKeeper >= 3.6.2 due to class loading issues --> <dependency> <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-core</artifactId> <scope>compile</scope> - <version>3.2.5</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -104,7 +103,6 @@ <groupId>org.xerial.snappy</groupId> <artifactId>snappy-java</artifactId> <scope>compile</scope> - <version>1.1.7</version> </dependency> <dependency> <groupId>org.mockito</groupId> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index bd3389b8d4d..7c91e54dd4b 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -36,7 +36,7 @@ public class MockCurator extends Curator { /** * Creates a mock curator * - * @param stableOrdering if true children of a node are returned in the same order each time they are queries. + * @param stableOrdering if true children of a node are returned in the same order each time they are queried. * This is not what ZooKeeper does. */ public MockCurator(boolean stableOrdering) { diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index a8105c78881..236bd5245a9 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -61,6 +61,11 @@ <artifactId>log4j-over-slf4j</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <scope>compile</scope> + </dependency> </dependencies> <build> diff --git a/zookeeper-server/zookeeper-server-3.7.0/pom.xml b/zookeeper-server/zookeeper-server-3.7.0/pom.xml index 01fd83a496b..8daa6003f1e 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/pom.xml +++ b/zookeeper-server/zookeeper-server-3.7.0/pom.xml @@ -62,7 +62,6 @@ <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-core</artifactId> <scope>compile</scope> - <version>3.2.5</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -74,7 +73,6 @@ <groupId>org.xerial.snappy</groupId> <artifactId>snappy-java</artifactId> <scope>compile</scope> - <version>1.1.7</version> </dependency> <dependency> <groupId>junit</groupId> |