diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-11-23 16:19:20 +0100 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2021-11-23 16:19:20 +0100 |
commit | cd91614db66b12e84293b8ba938cf2103aa95b28 (patch) | |
tree | 0ce87ab3b7bd57e9f13aec9fdae2a56baeb6d22b | |
parent | b342a8edb3c9ce28d53abbf89f70c1c3900a9db0 (diff) | |
parent | baa10a7a7762766d4b32d484bc21283cac543993 (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/supported-plans
289 files changed, 2331 insertions, 2187 deletions
diff --git a/abi-check-plugin/pom.xml b/abi-check-plugin/pom.xml index 7317205816c..059e593c0e9 100644 --- a/abi-check-plugin/pom.xml +++ b/abi-check-plugin/pom.xml @@ -40,7 +40,7 @@ <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> - <version>2.8.5</version> + <version>${gson.version}</version> </dependency> <dependency> <groupId>org.junit.jupiter</groupId> diff --git a/client/go/auth0/auth0.go b/client/go/auth0/auth0.go index 0cb613f13b7..377d56a1637 100644 --- a/client/go/auth0/auth0.go +++ b/client/go/auth0/auth0.go @@ -9,6 +9,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "os/signal" "path/filepath" @@ -16,7 +17,6 @@ import ( "sync" "time" - "github.com/joeshaw/envdecode" "github.com/lestrrat-go/jwx/jwt" "github.com/pkg/browser" "github.com/vespa-engine/vespa/client/go/auth" @@ -41,21 +41,19 @@ type System struct { type Auth0 struct { Authenticator *auth.Authenticator system string + systemApiUrl string initOnce sync.Once errOnce error Path string config config } -// default to vespa-cd.auth0.com -var ( - authCfg struct { - Audience string `env:"AUTH0_AUDIENCE,default=https://vespa-cd.auth0.com/api/v2/"` - ClientID string `env:"AUTH0_CLIENT_ID,default=4wYWA496zBP28SLiz0PuvCt8ltL11DZX"` - DeviceCodeEndpoint string `env:"AUTH0_DEVICE_CODE_ENDPOINT,default=https://vespa-cd.auth0.com/oauth/device/code"` - OauthTokenEndpoint string `env:"AUTH0_OAUTH_TOKEN_ENDPOINT,default=https://vespa-cd.auth0.com/oauth/token"` - } -) +type authCfg struct { + Audience string `json:"audience"` + ClientID string `json:"client-id"` + DeviceCodeEndpoint string `json:"device-code-endpoint"` + OauthTokenEndpoint string `json:"oauth-token-endpoint"` +} func ContextWithCancel() context.Context { ctx, cancel := context.WithCancel(context.Background()) @@ -71,22 +69,39 @@ func ContextWithCancel() context.Context { // GetAuth0 will try to initialize the config context, as well as figure out if // there's a readily available system. -func GetAuth0(configPath string, systemName string) (*Auth0, error) { +func GetAuth0(configPath string, systemName string, systemApiUrl string) (*Auth0, error) { a := Auth0{} a.Path = configPath a.system = systemName - if err := envdecode.StrictDecode(&authCfg); err != nil { - return nil, fmt.Errorf("could not decode env: %w", err) + a.systemApiUrl = systemApiUrl + c, err := a.getDeviceFlowConfig() + if err != nil { + return nil, fmt.Errorf("cannot get auth config: %w", err) } a.Authenticator = &auth.Authenticator{ - Audience: authCfg.Audience, - ClientID: authCfg.ClientID, - DeviceCodeEndpoint: authCfg.DeviceCodeEndpoint, - OauthTokenEndpoint: authCfg.OauthTokenEndpoint, + Audience: c.Audience, + ClientID: c.ClientID, + DeviceCodeEndpoint: c.DeviceCodeEndpoint, + OauthTokenEndpoint: c.OauthTokenEndpoint, } return &a, nil } +func (a *Auth0) getDeviceFlowConfig() (authCfg, error) { + systemApiUrl, _ := url.Parse(a.systemApiUrl + "/auth0/v1/device-flow-config") + r, err := http.Get(systemApiUrl.String()) + if err != nil { + return authCfg{}, fmt.Errorf("cannot get auth config: %w", err) + } + defer r.Body.Close() + var res authCfg + err = json.NewDecoder(r.Body).Decode(&res) + if err != nil { + return authCfg{}, fmt.Errorf("cannot decode response: %w", err) + } + return res, nil +} + // IsLoggedIn encodes the domain logic for determining whether we're // logged in. This might check our config storage, or just in memory. func (a *Auth0) IsLoggedIn() bool { diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index ba2df8c40dc..9832f04e3f0 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -48,6 +48,16 @@ var apiKeyCmd = &cobra.Command{ if err := ioutil.WriteFile(apiKeyFile, apiKey, 0600); err == nil { printSuccess("API private key written to ", apiKeyFile) printPublicKey(apiKeyFile, app.Tenant) + if vespa.Auth0AccessTokenEnabled() { + if err == nil { + if err := cfg.Set(cloudAuthFlag, "api-key"); err != nil { + fatalErr(err, "Could not write config") + } + if err := cfg.Write(); err != nil { + fatalErr(err) + } + } + } } else { fatalErr(err, "Failed to write ", apiKeyFile) } diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 62a97d9749d..750664e51b1 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -235,6 +235,12 @@ func (c *Config) Set(option, value string) error { viper.Set(option, value) return nil } + case cloudAuthFlag: + switch value { + case "access-token", "api-key": + viper.Set(option, value) + return nil + } } return fmt.Errorf("invalid option or value: %q: %q", option, value) } diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 084148f9bdc..79ba1fcef26 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -182,8 +182,8 @@ func getTarget() vespa.Target { return nil } var apiKey []byte = nil + apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant)) if !vespa.Auth0AccessTokenEnabled() { - apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant)) if err != nil { fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") } @@ -202,6 +202,20 @@ func getTarget() vespa.Target { if err != nil { fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'") } + var cloudAuth string + if vespa.Auth0AccessTokenEnabled() { + cloudAuth, err = cfg.Get(cloudAuthFlag) + if err != nil { + if apiKey != nil { + cloudAuth = "api-key" + } else { + cloudAuth = "access-token" + } + } + } else { + cloudAuth = "" + } + return vespa.CloudTarget(getApiURL(), deployment, apiKey, vespa.TLSOptions{ KeyPair: kp, @@ -213,7 +227,8 @@ func getTarget() vespa.Target { Level: vespa.LogLevel(logLevelArg), }, cfg.AuthConfigPath(), - getSystemName()) + getSystemName(), + cloudAuth) } fatalErrHint(fmt.Errorf("Invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL") return nil @@ -244,9 +259,9 @@ func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.T fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'") return opts } + var err error + opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) if !vespa.Auth0AccessTokenEnabled() { - var err error - opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) if err != nil { fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'") return opts diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go index 0e09a6d6244..f7b412a4613 100644 --- a/client/go/cmd/login.go +++ b/client/go/cmd/login.go @@ -24,11 +24,21 @@ var loginCmd = &cobra.Command{ if err != nil { return err } - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName()) + a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL()) if err != nil { return err } _, err = auth0.RunLogin(ctx, a, false) + if vespa.Auth0AccessTokenEnabled() { + if err == nil { + if err := cfg.Set(cloudAuthFlag, "access-token"); err != nil { + return err + } + if err := cfg.Write(); err != nil { + fatalErr(err) + } + } + } return err }, } diff --git a/client/go/cmd/logout.go b/client/go/cmd/logout.go index 18b7a2166ad..e3cfe6733eb 100644 --- a/client/go/cmd/logout.go +++ b/client/go/cmd/logout.go @@ -24,7 +24,7 @@ var logoutCmd = &cobra.Command{ if err != nil { return err } - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName()) + a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL()) if err != nil { return err } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 5aae55ab6e4..087be7c352d 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -51,6 +51,7 @@ const ( waitFlag = "wait" colorFlag = "color" quietFlag = "quiet" + cloudAuthFlag = "cloudAuth" ) func isTerminal() bool { diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index 9c5fb3a12ac..252a646bcfc 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -344,8 +344,10 @@ func checkDeploymentOpts(opts DeploymentOpts) error { if !opts.ApplicationPackage.HasCertificate() { return fmt.Errorf("%s: missing certificate in package", opts) } - if !Auth0AccessTokenEnabled() && opts.APIKey == nil { - return fmt.Errorf("%s: missing api key", opts.String()) + if !Auth0AccessTokenEnabled() { + if opts.APIKey == nil { + return fmt.Errorf("%s: missing api key", opts.String()) + } } return nil } diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index e4779e14c0d..093cb2b5cad 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -209,10 +209,34 @@ type cloudTarget struct { tlsOptions TLSOptions logOptions LogOptions - queryURL string - documentURL string + urlsByCluster map[string]string authConfigPath string systemName string + cloudAuth string +} + +func (t *cloudTarget) resolveEndpoint(cluster string) (string, error) { + if cluster == "" { + for _, u := range t.urlsByCluster { + if len(t.urlsByCluster) == 1 { + return u, nil + } else { + return "", fmt.Errorf("multiple clusters, none chosen: %v", t.urlsByCluster) + } + } + } else { + u := t.urlsByCluster[cluster] + if u == "" { + clusters := make([]string, len(t.urlsByCluster)) + for c := range t.urlsByCluster { + clusters = append(clusters, c) + } + return "", fmt.Errorf("unknown cluster '%s': must be one of %v", cluster, clusters) + } + return u, nil + } + + return "", fmt.Errorf("no endpoints") } func (t *cloudTarget) Type() string { return t.targetType } @@ -227,25 +251,37 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64) ( case deployService: return &Service{Name: name, BaseURL: t.apiURL}, nil case queryService: - if t.queryURL == "" { - return nil, fmt.Errorf("service %s is not discovered", name) + queryURL, err := t.resolveEndpoint("") + if err != nil { + return nil, err } - return &Service{Name: name, BaseURL: t.queryURL, TLSOptions: t.tlsOptions}, nil + return &Service{Name: name, BaseURL: queryURL, TLSOptions: t.tlsOptions}, nil case documentService: - if t.documentURL == "" { - return nil, fmt.Errorf("service %s is not discovered", name) + documentURL, err := t.resolveEndpoint("") + if err != nil { + return nil, err } - return &Service{Name: name, BaseURL: t.documentURL, TLSOptions: t.tlsOptions}, nil + return &Service{Name: name, BaseURL: documentURL, TLSOptions: t.tlsOptions}, nil } return nil, fmt.Errorf("unknown service: %s", name) } func (t *cloudTarget) PrepareApiRequest(req *http.Request, sigKeyId string) error { if Auth0AccessTokenEnabled() { - if err := t.addAuth0AccessToken(req); err != nil { - return err + if t.cloudAuth == "access-token" { + if err := t.addAuth0AccessToken(req); err != nil { + return err + } + } else { + if t.apiKey == nil { + return fmt.Errorf("Deployment to cloud requires an API key. Try 'vespa api-key'") + } + signer := NewRequestSigner(sigKeyId, t.apiKey) + if err := signer.SignRequest(req); err != nil { + return err + } } - } else if t.apiKey != nil { + } else { signer := NewRequestSigner(sigKeyId, t.apiKey) if err := signer.SignRequest(req); err != nil { return err @@ -255,7 +291,7 @@ func (t *cloudTarget) PrepareApiRequest(req *http.Request, sigKeyId string) erro } func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { - a, err := auth0.GetAuth0(t.authConfigPath, t.systemName) + a, err := auth0.GetAuth0(t.authConfigPath, t.systemName, t.apiURL) system, err := a.PrepareSystem(auth0.ContextWithCancel()) if err != nil { return err @@ -404,7 +440,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { if err := t.PrepareApiRequest(req, t.deployment.Application.SerializedForm()); err != nil { return err } - var endpointURL string + urlsByCluster := make(map[string]string) endpointFunc := func(status int, response []byte) (bool, error) { if ok, err := isOK(status); !ok { return ok, err @@ -416,17 +452,21 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { if len(resp.Endpoints) == 0 { return false, nil } - endpointURL = resp.Endpoints[0].URL + for _, endpoint := range resp.Endpoints { + if endpoint.Scope != "zone" { + continue + } + urlsByCluster[endpoint.Cluster] = endpoint.URL + } return true, nil } if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.tlsOptions.KeyPair, timeout); err != nil { return err } - if endpointURL == "" { - return fmt.Errorf("no endpoint discovered") + if len(urlsByCluster) == 0 { + return fmt.Errorf("no endpoints discovered") } - t.queryURL = endpointURL - t.documentURL = endpointURL + t.urlsByCluster = urlsByCluster return nil } @@ -448,7 +488,8 @@ func CustomTarget(baseURL string) Target { } // CloudTarget creates a Target for the Vespa Cloud platform. -func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions, authConfigPath string, systemName string) Target { +func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions, + authConfigPath string, systemName string, cloudAuth string) Target { return &cloudTarget{ apiURL: apiURL, targetType: cloudTargetType, @@ -458,11 +499,14 @@ func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions logOptions: logOptions, authConfigPath: authConfigPath, systemName: systemName, + cloudAuth: cloudAuth, } } type deploymentEndpoint struct { - URL string `json:"url"` + Cluster string `json:"cluster"` + URL string `json:"url"` + Scope string `json:"scope"` } type deploymentResponse struct { diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index 62bde3044bf..9d2418897e3 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -25,7 +25,7 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request case "/application/v4/tenant/t1/application/a1/instance/i1/environment/dev/region/us-north-1": response := "{}" if v.deploymentConverged { - response = fmt.Sprintf(`{"endpoints": [{"url": "%s"}]}`, v.serverURL) + response = fmt.Sprintf(`{"endpoints": [{"url": "%s","scope": "zone","cluster": "cluster1"}]}`, v.serverURL) } w.Write([]byte(response)) case "/application/v4/tenant/t1/application/a1/instance/i1/job/dev-us-north-1/run/42": @@ -152,7 +152,7 @@ func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { target := CloudTarget("https://example.com", Deployment{ Application: ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, Zone: ZoneID{Environment: "dev", Region: "us-north-1"}, - }, apiKey, TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: logWriter}, "", "") + }, apiKey, TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: logWriter}, "", "", "") if ct, ok := target.(*cloudTarget); ok { ct.apiURL = url } else { diff --git a/client/pom.xml b/client/pom.xml index 4abcdf9ac6c..3dee909b932 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -20,7 +20,7 @@ <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> - <version>2.8.5</version> + <version>${gson.version}</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 7113787a715..e2cc7085353 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -238,7 +238,7 @@ <include>net.java.dev.jna:jna:4.5.2:jar:test</include> <include>org.abego.treelayout:org.abego.treelayout.core:1.0.1:jar:test</include> <include>org.antlr:antlr-runtime:3.5.2:jar:test</include> - <include>org.antlr:antlr4-runtime:4.9.3:jar:test</include> + <include>org.antlr:antlr4-runtime:4.5:jar:test</include> <include>org.apache.commons:commons-exec:1.3:jar:test</include> <include>org.apache.commons:commons-math3:3.6.1:jar:test</include> <include>org.apache.httpcomponents:httpclient:4.5.12:jar:test</include> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 42041f3a0b0..6ed2f3daa30 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -108,6 +108,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return false; } + @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 6165cdd2dc1..b49fde9e5d7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -91,7 +91,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { } if ( ! schema.isDocumentsOnly()) { attributeFields = new AttributeFields(schema); - summaries = new Summaries(schema, deployLogger); + summaries = new Summaries(schema, deployLogger, deployProperties.featureFlags()); summaryMap = new SummaryMap(schema); juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankingConstants(), schema.rankExpressionFiles(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java index 5fdc51e8f5f..1455fbc92e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; @@ -14,9 +15,11 @@ import java.util.List; */ public class Summaries extends Derived implements SummaryConfig.Producer { + private final boolean useV8GeoPositions; private List<SummaryClass> summaries=new java.util.ArrayList<>(1); - public Summaries(Schema schema, DeployLogger deployLogger) { + public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) { + this.useV8GeoPositions = featureFlags.useV8GeoPositions(); // Make sure the default is first summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger)); for (DocumentSummary summary : schema.getSummaries().values()) { @@ -31,6 +34,7 @@ public class Summaries extends Derived implements SummaryConfig.Producer { @Override public void getConfig(SummaryConfig.Builder builder) { builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); + builder.usev8geopositions(useV8GeoPositions); for (SummaryClass summaryClass : summaries) { builder.classes(summaryClass.getSummaryClassConfig()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index a65e6fe16c0..f167a79fd05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -165,6 +165,7 @@ public class VespaMetricSet { metrics.add(new Metric("httpapi_succeeded.rate")); metrics.add(new Metric("httpapi_failed.rate")); metrics.add(new Metric("httpapi_parse_error.rate")); + addMetric(metrics, "httpapi_test_and_set_condition_not_met", List.of("rate")); metrics.add(new Metric("mem.heap.total.average")); metrics.add(new Metric("mem.heap.free.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 81dd458570b..f69b08ff300 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -114,8 +114,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler"; public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName(); public static final String LOG_HANDLER_CLASS = com.yahoo.container.handler.LogHandler.class.getName(); - public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1"; public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15"; + public static final String PARALLEL_GC = "-XX:+UseParallelGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1"; public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; public static final BindingPattern STATE_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(StateHandler.STATE_API_ROOT); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d65fbba6a5e..319f334ffc2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -8,7 +8,6 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.Endpoint; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ApplicationClusterEndpoint; @@ -661,15 +660,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { static boolean incompatibleGCOptions(String jvmargs) { Pattern gcAlgorithm = Pattern.compile("-XX:[-+]Use.+GC"); Pattern cmsArgs = Pattern.compile("-XX:[-+]*CMS"); - return (gcAlgorithm.matcher(jvmargs).find() ||cmsArgs.matcher(jvmargs).find()); + return (gcAlgorithm.matcher(jvmargs).find() || cmsArgs.matcher(jvmargs).find()); } - private static String buildJvmGCOptions(DeployState deployState, String jvmGCOPtions) { - String options = (jvmGCOPtions != null) - ? jvmGCOPtions + private static String buildJvmGCOptions(DeployState deployState, String jvmGCOptions) { + String options = (jvmGCOptions != null) + ? jvmGCOptions : deployState.getProperties().jvmGCOptions(); return (options == null || options.isEmpty()) - ? (deployState.isHosted() ? ContainerCluster.CMS : ContainerCluster.G1GC) + ? (deployState.isHosted() ? ContainerCluster.PARALLEL_GC : ContainerCluster.G1GC) : options; } @@ -685,7 +684,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } else { jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); if (incompatibleGCOptions(jvmOptions)) { - deployLogger.logApplicationPackage(WARNING, "You need to move out your GC related options from 'jvmargs' to 'jvm-gc-options'"); + deployLogger.logApplicationPackage(WARNING, "You need to move out your GC-related options from deprecated 'jvmargs' to 'jvm-gc-options'"); cluster.setJvmGCOptions(ContainerCluster.G1GC); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index c3851f2918a..d3d992c11f5 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// -------------------------------------------------------------------------------- -// -// JavaCC options. When this file is changed, run "mvn generate-sources" to rebuild -// the parser classes. + +// Schema parser. // -// -------------------------------------------------------------------------------- +// NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf + options { UNICODE_INPUT = true; CACHE_TOKENS = false; @@ -15,11 +14,6 @@ options { USER_CHAR_STREAM = true; } -// -------------------------------------------------------------------------------- -// -// Parser body. -// -// -------------------------------------------------------------------------------- PARSER_BEGIN(SDParser) package com.yahoo.searchdefinition.parser; diff --git a/config-model/src/test/derived/advanced/summary.cfg b/config-model/src/test/derived/advanced/summary.cfg index f497461b460..e4845c1994e 100644 --- a/config-model/src/test/derived/advanced/summary.cfg +++ b/config-model/src/test/derived/advanced/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1271952241 +usev8geopositions false classes[].id 1271952241 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg index 965c875d5ce..e750d1454e8 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 252850086 +usev8geopositions false classes[].id 252850086 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg index f0189f9a3c7..f52952e2871 100644 --- a/config-model/src/test/derived/attributeprefetch/summary.cfg +++ b/config-model/src/test/derived/attributeprefetch/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1151071433 +usev8geopositions false classes[].id 1151071433 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg index 2dac4736d23..1eeef44cd54 100644 --- a/config-model/src/test/derived/complex/summary.cfg +++ b/config-model/src/test/derived/complex/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1506848752 +usev8geopositions false classes[].id 1506848752 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/emptychild/summary.cfg b/config-model/src/test/derived/emptychild/summary.cfg index 82bed7fd55e..733fa1fde54 100644 --- a/config-model/src/test/derived/emptychild/summary.cfg +++ b/config-model/src/test/derived/emptychild/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1814603381 +usev8geopositions false classes[].id 1814603381 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/emptydefault/summary.cfg b/config-model/src/test/derived/emptydefault/summary.cfg index 61294d97b4c..5aacec1d0b6 100644 --- a/config-model/src/test/derived/emptydefault/summary.cfg +++ b/config-model/src/test/derived/emptydefault/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1151071433 +usev8geopositions false classes[].id 1151071433 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/id/summary.cfg b/config-model/src/test/derived/id/summary.cfg index b50b970afe2..f7e9f6a239f 100644 --- a/config-model/src/test/derived/id/summary.cfg +++ b/config-model/src/test/derived/id/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1814716401 +usev8geopositions false classes[].id 1814716401 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg index 3ab8e7e29e5..722443641cd 100644 --- a/config-model/src/test/derived/imported_position_field/summary.cfg +++ b/config-model/src/test/derived/imported_position_field/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1570252291 +usev8geopositions false classes[].id 1570252291 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/imported_position_field_summary/summary.cfg b/config-model/src/test/derived/imported_position_field_summary/summary.cfg index 76faac23170..0642382aabe 100644 --- a/config-model/src/test/derived/imported_position_field_summary/summary.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1194448774 +usev8geopositions false classes[].id 1194448774 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/imported_struct_fields/summary.cfg b/config-model/src/test/derived/imported_struct_fields/summary.cfg index ab6c6853925..0a9b29524e1 100644 --- a/config-model/src/test/derived/imported_struct_fields/summary.cfg +++ b/config-model/src/test/derived/imported_struct_fields/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1570252291 +usev8geopositions false classes[].id 1570252291 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/importedfields/summary.cfg b/config-model/src/test/derived/importedfields/summary.cfg index 74b5b44214e..975c601a97d 100644 --- a/config-model/src/test/derived/importedfields/summary.cfg +++ b/config-model/src/test/derived/importedfields/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1294344677 +usev8geopositions false classes[].id 1294344677 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/indexswitches/summary.cfg b/config-model/src/test/derived/indexswitches/summary.cfg index d04bc4eb167..bcc050dad4f 100644 --- a/config-model/src/test/derived/indexswitches/summary.cfg +++ b/config-model/src/test/derived/indexswitches/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1698765342 +usev8geopositions false classes[].id 1698765342 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/inheritance/summary.cfg b/config-model/src/test/derived/inheritance/summary.cfg index dde9f95ecbe..73c22f82a99 100644 --- a/config-model/src/test/derived/inheritance/summary.cfg +++ b/config-model/src/test/derived/inheritance/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1797992819 +usev8geopositions false classes[].id 1797992819 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/integerattributetostringindex/summary.cfg b/config-model/src/test/derived/integerattributetostringindex/summary.cfg index d5eb316ff01..bfbb1eb01ca 100644 --- a/config-model/src/test/derived/integerattributetostringindex/summary.cfg +++ b/config-model/src/test/derived/integerattributetostringindex/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1195656216 +usev8geopositions false classes[].id 1195656216 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/mail/summary.cfg b/config-model/src/test/derived/mail/summary.cfg index 3628e1acb88..2b886a4622b 100644 --- a/config-model/src/test/derived/mail/summary.cfg +++ b/config-model/src/test/derived/mail/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1831052622 +usev8geopositions false classes[].id 1831052622 classes[].name "default" classes[].fields[].name "snippet" diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg index b465bdfa541..b50b40b9b72 100644 --- a/config-model/src/test/derived/map_attribute/summary.cfg +++ b/config-model/src/test/derived/map_attribute/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1376056200 +usev8geopositions false classes[].id 1376056200 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg index 67988dbf30e..cf875704fa0 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1131098132 +usev8geopositions false classes[].id 1131098132 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/mlr/summary.cfg b/config-model/src/test/derived/mlr/summary.cfg index b6a53a9a1d9..5bea06a8472 100644 --- a/config-model/src/test/derived/mlr/summary.cfg +++ b/config-model/src/test/derived/mlr/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1868876861 +usev8geopositions false classes[].id 1868876861 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index 8d85890b156..16d0024155b 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 235127765 +usev8geopositions false classes[1156201411].id 1156201411 classes[1156201411].name "attributeprefetch" classes[1156201411].fields[a].name "a" diff --git a/config-model/src/test/derived/music/summary.cfg b/config-model/src/test/derived/music/summary.cfg index bc55727b407..c92b5491ebc 100644 --- a/config-model/src/test/derived/music/summary.cfg +++ b/config-model/src/test/derived/music/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 2086497905 +usev8geopositions false classes[].id 2086497905 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/newrank/summary.cfg b/config-model/src/test/derived/newrank/summary.cfg index 0b98b20c342..ddebcbe6ca6 100644 --- a/config-model/src/test/derived/newrank/summary.cfg +++ b/config-model/src/test/derived/newrank/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 912980235 +usev8geopositions false classes[].id 912980235 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg index 4222e88cc2f..cd7c295ab11 100644 --- a/config-model/src/test/derived/position_nosummary/summary.cfg +++ b/config-model/src/test/derived/position_nosummary/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1727020212 +usev8geopositions false classes[].id 1727020212 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg index f54066d865e..7b8bf16287f 100644 --- a/config-model/src/test/derived/position_summary/summary.cfg +++ b/config-model/src/test/derived/position_summary/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 230670304 +usev8geopositions false classes[].id 230670304 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/predicate_attribute/summary.cfg b/config-model/src/test/derived/predicate_attribute/summary.cfg index 9cc613107e0..10040fd71c6 100644 --- a/config-model/src/test/derived/predicate_attribute/summary.cfg +++ b/config-model/src/test/derived/predicate_attribute/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1391971216 +usev8geopositions false classes[].id 1391971216 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg index 22f4c3d4ca7..aec076aa8fe 100644 --- a/config-model/src/test/derived/rankexpression/summary.cfg +++ b/config-model/src/test/derived/rankexpression/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1753207254 +usev8geopositions false classes[].id 1753207254 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/ranktypes/summary.cfg b/config-model/src/test/derived/ranktypes/summary.cfg index 49b668e9edf..7ab01302d8b 100644 --- a/config-model/src/test/derived/ranktypes/summary.cfg +++ b/config-model/src/test/derived/ranktypes/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1567556360 +usev8geopositions false classes[].id 1567556360 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/reference_fields/summary.cfg b/config-model/src/test/derived/reference_fields/summary.cfg index 410bccff7b3..b23b6c5b5a8 100644 --- a/config-model/src/test/derived/reference_fields/summary.cfg +++ b/config-model/src/test/derived/reference_fields/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1987541865 +usev8geopositions false classes[].id 1987541865 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/schemainheritance/summary.cfg b/config-model/src/test/derived/schemainheritance/summary.cfg index d3286961007..6fcf5b2aaa8 100644 --- a/config-model/src/test/derived/schemainheritance/summary.cfg +++ b/config-model/src/test/derived/schemainheritance/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1313596701 +usev8geopositions false classes[].id 1313596701 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/streamingstruct/summary.cfg b/config-model/src/test/derived/streamingstruct/summary.cfg index 655499a88be..d7b7057bf1f 100644 --- a/config-model/src/test/derived/streamingstruct/summary.cfg +++ b/config-model/src/test/derived/streamingstruct/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 569269436 +usev8geopositions false classes[].id 569269436 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/streamingstructdefault/summary.cfg b/config-model/src/test/derived/streamingstructdefault/summary.cfg index a52b34925dc..7274804a5de 100644 --- a/config-model/src/test/derived/streamingstructdefault/summary.cfg +++ b/config-model/src/test/derived/streamingstructdefault/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 718801936 +usev8geopositions false classes[].id 718801936 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg index 355cba0e561..de6a2a6e386 100644 --- a/config-model/src/test/derived/tensor/summary.cfg +++ b/config-model/src/test/derived/tensor/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 898020074 +usev8geopositions false classes[].id 898020074 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/derived/twostreamingstructs/summary.cfg b/config-model/src/test/derived/twostreamingstructs/summary.cfg index 28f19e6fe25..969e91d4b87 100644 --- a/config-model/src/test/derived/twostreamingstructs/summary.cfg +++ b/config-model/src/test/derived/twostreamingstructs/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 569269436 +usev8geopositions false classes[].id 569269436 classes[].name "default" classes[].fields[].name "coupleof" diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg index e0e67a5669d..fc3b9e655ca 100644 --- a/config-model/src/test/derived/types/summary.cfg +++ b/config-model/src/test/derived/types/summary.cfg @@ -1,4 +1,5 @@ defaultsummaryid 1131946680 +usev8geopositions false classes[].id 1131946680 classes[].name "default" classes[].omitsummaryfeatures false diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 9fa86d38142..9fda6016969 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -132,14 +132,14 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { @Test public void requireThatJvmGCOptionsIsHonoured() throws IOException, SAXException { - verifyJvmGCOptions(false, null,null, ContainerCluster.G1GC); - verifyJvmGCOptions(true, null,null, ContainerCluster.CMS); - verifyJvmGCOptions(true, "",null, ContainerCluster.CMS); - verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC"); - verifyJvmGCOptions(true, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC"); - verifyJvmGCOptions(false, null,"-XX:+UseG1GC", "-XX:+UseG1GC"); - verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC","-XX:+UseG1GC", "-XX:+UseG1GC"); - verifyJvmGCOptions(false, null,"-XX:+UseConcMarkSweepGC", "-XX:+UseConcMarkSweepGC"); + verifyJvmGCOptions(false, null, null, ContainerCluster.G1GC); + verifyJvmGCOptions(true, null, null, ContainerCluster.PARALLEL_GC); + verifyJvmGCOptions(true, "", null, ContainerCluster.PARALLEL_GC); + verifyJvmGCOptions(false, "-XX:+UseG1GC", null, "-XX:+UseG1GC"); + verifyJvmGCOptions(true, "-XX:+UseG1GC", null, "-XX:+UseG1GC"); + verifyJvmGCOptions(false, null, "-XX:+UseG1GC", "-XX:+UseG1GC"); + verifyJvmGCOptions(false, "-XX:+UseParallelGC", "-XX:+UseG1GC", "-XX:+UseG1GC"); + verifyJvmGCOptions(false, null, "-XX:+UseParallelGC", "-XX:+UseParallelGC"); } } diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def index 5169df7d72e..9b231d86c38 100644 --- a/configdefinitions/src/vespa/summary.def +++ b/configdefinitions/src/vespa/summary.def @@ -2,6 +2,9 @@ namespace=vespa.config.search defaultsummaryid int default=-1 + +usev8geopositions bool default=false + classes[].id int classes[].name string classes[].omitsummaryfeatures bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index ea823e5befb..a7deac443a5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -200,6 +200,7 @@ public class ModelContextImpl implements ModelContext { private final boolean asyncApplyBucketDiff; private final boolean ignoreThreadStackSizes; private final boolean unorderedMergeChaining; + private final boolean useV8GeoPositions; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -238,6 +239,8 @@ public class ModelContextImpl implements ModelContext { this.asyncApplyBucketDiff = flagValue(source, appId, Flags.ASYNC_APPLY_BUCKET_DIFF); this.ignoreThreadStackSizes = flagValue(source, appId, Flags.IGNORE_THREAD_STACK_SIZES); this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); + this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); + } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -278,6 +281,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean ignoreThreadStackSizes() { return ignoreThreadStackSizes; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } + @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 779a5f65673..aedbb3afb69 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -100,6 +100,8 @@ class HttpRequestDispatch { if (t != null) requestCompletion.completeExceptionally(t); else requestCompletion.complete(null); }); + // Start the reader after wiring of "finished futures" are complete + servletRequestReader.start(); } ContentChannel dispatchFilterRequest(Response response) { @@ -217,11 +219,7 @@ class HttpRequestDispatch { HttpRequestFactory.copyHeaders(jettyRequest, jdiscRequest); requestContentChannel = requestHandler.handleRequest(jdiscRequest, servletResponseController.responseHandler()); } - //TODO If the below method throws servletRequestReader will not complete and - // requestContentChannel will not be closed and there is a reference leak - // Ditto for the servletInputStream - return new ServletRequestReader( - jettyRequest.getInputStream(), requestContentChannel, jDiscContext.janitor, metricReporter); + return new ServletRequestReader(jettyRequest, requestContentChannel, jDiscContext.janitor, metricReporter); } private static RequestHandler newRequestHandler(JDiscContext context, diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 1def9ccaab1..43050a53f58 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -6,6 +6,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import javax.servlet.ReadListener; import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; @@ -33,7 +34,7 @@ import java.util.logging.Logger; class ServletRequestReader { private enum State { - READING, ALL_DATA_READ, REQUEST_CONTENT_CLOSED + NOT_STARTED, READING, ALL_DATA_READ, REQUEST_CONTENT_CLOSED } private static final Logger log = Logger.getLogger(ServletRequestReader.class.getName()); @@ -42,11 +43,12 @@ class ServletRequestReader { private final Object monitor = new Object(); - private final ServletInputStream in; + private final HttpServletRequest req; private final ContentChannel requestContentChannel; private final Janitor janitor; private final RequestMetricReporter metricReporter; + private ServletInputStream in; private Throwable errorDuringRead; private int bytesRead; @@ -63,7 +65,7 @@ class ServletRequestReader { * (i.e. when being called from user code, don't call back into user code.) */ // GuardedBy("monitor") - private State state = State.READING; + private State state = State.NOT_STARTED; /** * Number of calls that we're waiting for from user code. @@ -94,15 +96,31 @@ class ServletRequestReader { private final CompletableFuture<Void> finishedFuture = new CompletableFuture<>(); ServletRequestReader( - ServletInputStream in, + HttpServletRequest req, ContentChannel requestContentChannel, Janitor janitor, RequestMetricReporter metricReporter) { - this.in = Objects.requireNonNull(in); + this.req = Objects.requireNonNull(req); this.requestContentChannel = Objects.requireNonNull(requestContentChannel); this.janitor = Objects.requireNonNull(janitor); this.metricReporter = Objects.requireNonNull(metricReporter); - in.setReadListener(new Listener()); + } + + /** Register read listener to start reading request data */ + void start() { + try { + ServletInputStream in; + synchronized (monitor) { + if (state != State.NOT_STARTED) throw new IllegalStateException("State=" + state); + in = req.getInputStream(); // may throw + this.in = in; + state = State.READING; + } + // Not holding monitor in case listener is invoked from this thread + in.setReadListener(new Listener()); // may throw + } catch (Throwable t) { + fail(t); + } } CompletableFuture<Void> finishedFuture() { return finishedFuture; } @@ -111,6 +129,8 @@ class ServletRequestReader { @Override public void onDataAvailable() throws IOException { + ServletInputStream in; + synchronized (monitor) { in = ServletRequestReader.this.in; } while (in.isReady()) { final byte[] buffer = new byte[BUFFER_SIZE_BYTES]; int numBytesRead; diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java new file mode 100644 index 00000000000..1784aa77966 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.yql; + +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.CharStream; + +/** + * This class supports case-insensitive lexing by wrapping an existing + * {@link CharStream} and forcing the lexer to see only lowercase characters. + * Grammar literals should then be only lower case such as 'begin'. The text of the character + * stream is unaffected. Example: input 'BeGiN' would match lexer rule + * 'begin', but getText() would return 'BeGiN'. + * It is based on https://github.com/antlr/antlr4/blob/master/doc/resources/CaseChangingCharStream.java + */ +class CaseInsensitiveCharStream implements CharStream { + + final CharStream stream; + + /** + * Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing + * all characters lower case. + * @param stream The stream to wrap. + */ + CaseInsensitiveCharStream(CharStream stream) { + this.stream = stream; + } + + @Override + public String getText(Interval interval) { + return stream.getText(interval); + } + + @Override + public void consume() { + stream.consume(); + } + + @Override + public int LA(int i) { + int c = stream.LA(i); + if (c <= 0) { + return c; + } + return Character.toLowerCase(c); + } + + @Override + public int mark() { + return stream.mark(); + } + + @Override + public void release(int marker) { + stream.release(marker); + } + + @Override + public int index() { + return stream.index(); + } + + @Override + public void seek(int index) { + stream.seek(index); + } + + @Override + public int size() { + return stream.size(); + } + + @Override + public String getSourceName() { + return stream.getSourceName(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java deleted file mode 100644 index 5e21ce234d8..00000000000 --- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.yql; - -import org.antlr.v4.runtime.ANTLRFileStream; -import org.antlr.v4.runtime.CharStream; - -import java.io.IOException; - -/** - * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token. - */ - -class CaseInsensitiveFileStream extends ANTLRFileStream { - - public CaseInsensitiveFileStream(String fileName) throws IOException { - super(fileName); - } - - public CaseInsensitiveFileStream(String fileName, String encoding) throws IOException { - super(fileName, encoding); - } - - @Override - public int LA(int i) { - if (i == 0) { - return 0; - } - if (i < 0) { - i++; // e.g., translate LA(-1) to use offset 0 - } - - if ((p + i - 1) >= n) { - return CharStream.EOF; - } - return Character.toLowerCase(data[p + i - 1]); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java deleted file mode 100644 index ed071370f4e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.yql; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token. - */ -class CaseInsensitiveInputStream extends ANTLRInputStream { - - public CaseInsensitiveInputStream() { - super(); - } - - public CaseInsensitiveInputStream(InputStream input) throws IOException { - super(input); - } - - public CaseInsensitiveInputStream(InputStream input, int size) throws IOException { - super(input, size); - } - - public CaseInsensitiveInputStream(char[] data, int numberOfActualCharsInArray) throws IOException { - super(data, numberOfActualCharsInArray); - } - - public CaseInsensitiveInputStream(String input) throws IOException { - super(input); - } - - @Override - public int LA(int i) { - if (i == 0) { - return 0; - } - if (i < 0) { - i++; // e.g., translate LA(-1) to use offset 0 - } - - if ((p + i - 1) >= n) { - return CharStream.EOF; - } - return Character.toLowerCase(data[p + i - 1]); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index d32033249f1..c84ed0a0565 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -56,6 +56,7 @@ import com.yahoo.search.yql.yqlplusParser.TimeoutContext; import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext; import com.yahoo.search.yql.yqlplusParser.WhereContext; +import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; @@ -69,7 +70,6 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.TerminalNode; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; @@ -83,50 +83,39 @@ import java.util.Set; final class ProgramParser { public yqlplusParser prepareParser(String programName, InputStream input) throws IOException { - return prepareParser(programName, new CaseInsensitiveInputStream(input)); + //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading + return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input))); } public yqlplusParser prepareParser(String programName, String input) throws IOException { - return prepareParser(programName, new CaseInsensitiveInputStream(input)); + //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading + return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input))); } - public yqlplusParser prepareParser(File file) throws IOException { - return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath())); + private static class ErrorListener extends BaseErrorListener { + private final String programName; + ErrorListener(String programName) { this.programName = programName; } + @Override + public void syntaxError(Recognizer<?, ?> recognizer, + Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); + } } private yqlplusParser prepareParser(String programName, CharStream input) { + ErrorListener errorListener = new ErrorListener(programName); yqlplusLexer lexer = new yqlplusLexer(input); lexer.removeErrorListeners(); - lexer.addErrorListener(new BaseErrorListener() { - - @Override - public void syntaxError(Recognizer<?, ?> recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); - } - - }); + lexer.addErrorListener(errorListener); TokenStream tokens = new CommonTokenStream(lexer); yqlplusParser parser = new yqlplusParser(tokens); parser.removeErrorListeners(); - parser.addErrorListener(new BaseErrorListener() { - - @Override - public void syntaxError(Recognizer<?, ?> recognizer, - Object offendingSymbol, - int line, - int charPositionInLine, - String msg, - RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); - } - - }); + parser.addErrorListener(errorListener); parser.getInterpreter().setPredictionMode(PredictionMode.SLL); return parser; } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index f9005387716..ab99118b179 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -4,15 +4,14 @@ package com.yahoo.search.yql; import com.yahoo.component.chain.Chain; import com.yahoo.container.QrSearchersConfig; import com.yahoo.language.Language; -import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.BoolItem; -import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.PhraseItem; @@ -28,6 +27,7 @@ import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; @@ -42,7 +42,6 @@ import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.query.Sorting.UcaSorter; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; - import com.yahoo.search.searchchain.Execution; import org.junit.Test; @@ -71,6 +70,13 @@ public class YqlParserTestCase { private final YqlParser parser = new YqlParser(new ParserEnvironment()); @Test + public void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { + assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10;", + new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 " + + "no viable alternative at input 'rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d''")); + } + + @Test public void testParserDefaults() { assertTrue(parser.isQueryParser()); assertNull(parser.getDocTypes()); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java index 55a7af45fd2..2f1b93158ab 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java @@ -50,9 +50,10 @@ public class EndpointStatus { } /** - * @return The epoch for when this status became active + * @return The epoch for when this status became active, in seconds */ public long getEpoch() { return epoch; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index f3368d0918b..9e7c614d4e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -351,6 +351,7 @@ public class ApplicationController { TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); ZoneId zone = job.type().zone(controller.system()); + DeploymentId deployment = new DeploymentId(job.application(), zone); try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { Set<ContainerEndpoint> containerEndpoints; @@ -364,7 +365,7 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); - ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(new DeploymentId(job.application(), zone), revision)); + ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); @@ -376,8 +377,7 @@ public class ApplicationController { applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone, applicationPackage.deploymentSpec()); - - containerEndpoints = controller.routing().containerEndpointsOf(application, job.application().instance(), zone); + containerEndpoints = controller.routing().of(deployment).prepare(application); } // Release application lock while doing the deployment, which is a lengthy task. @@ -391,7 +391,7 @@ public class ApplicationController { // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as // the source since it's the same application, so it should have the same warnings NotificationSource source = zone.environment().isManuallyDeployed() ? - NotificationSource.from(new DeploymentId(job.application(), zone)) : NotificationSource.from(applicationId); + NotificationSource.from(deployment) : NotificationSource.from(applicationId); List<String> warnings = Optional.ofNullable(result.prepareResponse().log) .map(logs -> logs.stream() .filter(log -> log.applicationPackage) @@ -476,6 +476,7 @@ public class ApplicationController { ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, boolean dryRun) { + DeploymentId deployment = new DeploymentId(application, zone); try { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag @@ -490,7 +491,7 @@ public class ApplicationController { .map(tenant -> ((AthenzTenant)tenant).domain()); if (zone.environment().isManuallyDeployed()) - controller.applications().applicationStore().putMeta(new DeploymentId(application, zone), + controller.applications().applicationStore().putMeta(deployment, clock.instant(), applicationPackage.metaDataZip()); @@ -502,9 +503,9 @@ public class ApplicationController { .filter(tenant-> tenant instanceof CloudTenant) .map(tenant -> ((CloudTenant) tenant).tenantSecretStores()) .orElse(List.of()); - List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(new DeploymentId(application, zone)).stream() - .map(SupportAccessGrant::certificate) - .collect(toList()); + List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() + .map(SupportAccessGrant::certificate) + .collect(toList()); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, @@ -515,9 +516,10 @@ public class ApplicationController { return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { - // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that - // any DNS updates can be propagated as early as possible. - controller.routing().policies().refresh(application, applicationPackage.deploymentSpec(), zone); + // Even if prepare fails, routing configuration may need to be updated + if ( ! application.instance().isTester()) { + controller.routing().of(deployment).configure(applicationPackage.deploymentSpec()); + } } } @@ -701,7 +703,7 @@ public class ApplicationController { try { configServer.deactivate(id); } finally { - controller.routing().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); + controller.routing().of(id).configure(application.get().deploymentSpec()); if (zone.environment().isManuallyDeployed()) applicationStore.putMetaTombstone(id, clock.instant()); if (!zone.environment().isTest()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index ea2bcfcac4b..6e31c93dbdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import java.time.Instant; import java.util.Collection; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 3794b69c023..4772dbeaab1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -18,7 +18,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -31,10 +30,16 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.ExclusiveDeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.SharedDeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.ExclusiveRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.SharedRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import java.nio.charset.StandardCharsets; @@ -44,7 +49,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -77,6 +81,25 @@ public class RoutingController { this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource()); } + /** Create a routing context for given deployment */ + public DeploymentRoutingContext of(DeploymentId deployment) { + if (usesSharedRouting(deployment.zoneId())) { + return new SharedDeploymentRoutingContext(deployment, + this, + controller.serviceRegistry().configServer(), + controller.clock()); + } + return new ExclusiveDeploymentRoutingContext(deployment, this); + } + + /** Create a routing context for given zone */ + public RoutingContext of(ZoneId zone) { + if (usesSharedRouting(zone)) { + return new SharedRoutingContext(zone, controller.serviceRegistry().configServer()); + } + return new ExclusiveRoutingContext(zone, routingPolicies); + } + public RoutingPolicies policies() { return routingPolicies; } @@ -145,16 +168,17 @@ public class RoutingController { .collect(Collectors.toMap(t -> new DeploymentId(application.id().instance(t.instance()), ZoneId.from(Environment.prod, t.region())), t -> t.weight())); - List<RoutingMethod> availableRoutingMethods = routingMethodsOfAll(deployments.keySet(), deploymentSpec); - for (var routingMethod : availableRoutingMethods) { - endpoints.add(Endpoint.of(application.id()) - .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), - ClusterSpec.Id.from(declaredEndpoint.containerId()), - deployments) - .routingMethod(routingMethod) - .on(Port.fromRoutingMethod(routingMethod)) - .in(controller.system())); - } + // An application endpoint can only target a single zone, so we just pick the zone of any deployment target + ZoneId zone = deployments.keySet().iterator().next().zoneId(); + // Application endpoints are only supported when using direct routing methods + RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive; + endpoints.add(Endpoint.of(application.id()) + .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), + ClusterSpec.Id.from(declaredEndpoint.containerId()), + deployments) + .routingMethod(routingMethod) + .on(Port.fromRoutingMethod(routingMethod)) + .in(controller.system())); } return EndpointList.copyOf(endpoints); } @@ -216,43 +240,6 @@ public class RoutingController { return Collections.unmodifiableList(endpointDnsNames); } - /** Change status of all global endpoints for given deployment */ - public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { - readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { - try { - controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), status); - } catch (Exception e) { - throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); - } - }); - } - - /** Get global endpoint status for given deployment */ - public Map<Endpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { - var routingEndpoints = new LinkedHashMap<Endpoint, EndpointStatus>(); - readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { - var upstreamName = endpoint.upstreamIdOf(deployment); - var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, upstreamName); - routingEndpoints.put(endpoint, status); - }); - return Collections.unmodifiableMap(routingEndpoints); - } - - /** - * Assigns one or more global rotations to given application, if eligible. The given application is implicitly - * stored, ensuring that the assigned rotation(s) are persisted when this returns. - */ - private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { - try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), - application.get().require(instanceName), - rotationLock); - application = application.with(instanceName, instance -> instance.with(rotations)); - controller.applications().store(application); // store assigned rotation even if deployment fails - } - return application; - } - /** Returns the global and application-level endpoints for given deployment, as container endpoints */ public Set<ContainerEndpoint> containerEndpointsOf(LockedApplication application, InstanceName instanceName, ZoneId zone) { // Assign rotations to application @@ -354,6 +341,36 @@ public class RoutingController { Priority.normal)); } + /** Returns direct routing endpoints if any exist and feature flag is set for given application */ + // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant + public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) { + boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); + EndpointList directEndpoints = endpoints.direct(); + if (hideSharedEndpoint && !directEndpoints.isEmpty()) { + return directEndpoints; + } + return endpoints; + } + + /** + * Assigns one or more global rotations to given application, if eligible. The given application is implicitly + * stored, ensuring that the assigned rotation(s) are persisted when this returns. + */ + private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { + try (RotationLock rotationLock = rotationRepository.lock()) { + var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), + application.get().require(instanceName), + rotationLock); + application = application.with(instanceName, instance -> instance.with(rotations)); + controller.applications().store(application); // store assigned rotation even if deployment fails + } + return application; + } + + private boolean usesSharedRouting(ZoneId zone) { + return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); + } + /** Returns the routing methods that are available across all given deployments */ private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) { var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>(); @@ -437,23 +454,12 @@ public class RoutingController { } /** Create a common name based on a hash of given application. This must be less than 64 characters long. */ - private String commonNameHashOf(ApplicationId application, SystemName system) { + private static String commonNameHashOf(ApplicationId application, SystemName system) { HashCode sha1 = Hashing.sha1().hashString(application.serializedForm(), StandardCharsets.UTF_8); String base32 = BaseEncoding.base32().omitPadding().lowerCase().encode(sha1.asBytes()); return 'v' + base32 + Endpoint.internalDnsSuffix(system); } - /** Returns direct routing endpoints if any exist and feature flag is set for given application */ - // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant - public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) { - boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - EndpointList directEndpoints = endpoints.direct(); - if (hideSharedEndpoint && !directEndpoints.isEmpty()) { - return directEndpoints; - } - return endpoints; - } - private static String asString(Endpoint.Scope scope) { switch (scope) { case application: return "application"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 1596456b7cc..ab9304e75f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import java.util.Collection; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index c736863a57e..aee7c1052be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -60,7 +60,7 @@ public class Endpoint { this.instance = requireInstance(instanceName, scope); this.url = url; this.targets = List.copyOf(requireTargets(targets, application, instanceName, scope, certificateName)); - this.scope = scope; + this.scope = requireScope(scope, routingMethod); this.legacy = legacy; this.routingMethod = routingMethod; this.tls = port.tls; @@ -329,6 +329,11 @@ public class Endpoint { return instanceName; } + private static Scope requireScope(Scope scope, RoutingMethod routingMethod) { + if (scope == Scope.application && !routingMethod.isDirect()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not support " + scope + "-scoped endpoints"); + return scope; + } + private static List<Target> requireTargets(List<Target> targets, TenantAndApplicationId application, Optional<InstanceName> instanceName, Scope scope, boolean certificateName) { if (!certificateName && targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); if (scope == Scope.zone && targets.size() != 1) throw new IllegalArgumentException("Exactly one target must be given for " + scope + " endpoints"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 4fcd6b10efa..9789bbd4da2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -41,15 +41,16 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ActivateResult; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; @@ -477,12 +478,12 @@ public class InternalStepRunner implements StepRunner { } private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zone))); + DeploymentId deployment = new DeploymentId(id, zone); + Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readZoneEndpointsOf(Set.of(deployment)); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; } - var policies = controller.routing().policies().get(new DeploymentId(id, zone)); for (var endpoint : endpoints.get(zone)) { HostName endpointName = HostName.from(endpoint.dnsName()); var ipAddress = controller.jobController().cloud().resolveHostName(endpointName); @@ -490,10 +491,10 @@ public class InternalStepRunner implements StepRunner { logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'."); return false; } - if (endpoint.routingMethod() == RoutingMethod.exclusive) { - var policy = policies.get(new RoutingPolicyId(id, ClusterSpec.Id.from(endpoint.name()), zone)); - if (policy == null) - throw new IllegalStateException(endpoint + " has no matching policy in " + policies); + DeploymentRoutingContext context = controller.routing().of(deployment); + if (context.routingMethod() == RoutingMethod.exclusive) { + RoutingPolicy policy = context.routingPolicy(ClusterSpec.Id.from(endpoint.name())) + .orElseThrow(() -> new IllegalStateException(endpoint + " has no matching policy")); var cNameValue = controller.jobController().cloud().resolveCname(endpointName); if ( ! cNameValue.map(policy.canonicalName()::equals).orElse(false)) { @@ -848,11 +849,9 @@ public class InternalStepRunner implements StepRunner { ZoneId zone = id.type().zone(controller.system()); boolean useTesterCertificate = useTesterCertificate(id); - boolean useOsgiBasedTestRuntime = testerPlatformVersion(id).isAfter(new Version(7, 247, 11)); byte[] servicesXml = servicesXml( ! controller.system().isPublic(), useTesterCertificate, - useOsgiBasedTestRuntime, testerResourcesFor(zone, spec.requireInstance(id.application().instance())), controller.controllerConfig().steprunner().testerapp()); byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version); @@ -904,9 +903,8 @@ public class InternalStepRunner implements StepRunner { } /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml( - boolean systemUsesAthenz, boolean useTesterCertificate, boolean useOsgiBasedTestRuntime, - NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { + static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, + NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); @@ -920,24 +918,6 @@ public class InternalStepRunner implements StepRunner { String runtimeProviderClass = config.runtimeProviderClass(); String tenantCdBundle = config.tenantCdBundle(); - String extraJUnitComponents = - "\n" + - " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n" + - "\n" + - " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + - " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + - " <artifactsPath>artifacts</artifactsPath>\n" + - " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + - " </config>\n" + - " </component>\n"; - String servicesXml = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<services xmlns:deploy='vespa' version='1.0'>\n" + @@ -955,7 +935,22 @@ public class InternalStepRunner implements StepRunner { " <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" + " <binding>http://*/tester/v1/*</binding>\n" + " </handler>\n" + - (useOsgiBasedTestRuntime ? extraJUnitComponents : "") + + "\n" + + " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + + " </component>\n" + + "\n" + + " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" + + " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" + + " <artifactsPath>artifacts</artifactsPath>\n" + + " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" + + " </config>\n" + + " </component>\n" + "\n" + " <nodes count=\"1\" allocated-memory=\"" + jdiscMemoryPct + "%\">\n" + " " + resourceString + "\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 6ba8ad5bf36..ebda767d79e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -578,16 +578,8 @@ public class JobController { }); } - // TODO(mpolden): Eliminate duplication in this and ApplicationController#deactivate public void deactivateTester(TesterId id, JobType type) { - var zone = type.zone(controller.system()); - try { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), zone)); - } finally { - // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and - // tester instances have none. - controller.routing().policies().refresh(id.id(), DeploymentSpec.empty, zone); - } + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); } private void prunePackages(TenantAndApplicationId id) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 47df7a9da92..2939d10f99e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index 1d5d444a32c..5acb21917eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; @@ -25,7 +26,8 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { for (var zone : controller().zoneRegistry().zones().reachable().ids()) { for (var application : SystemApplication.values()) { if (!application.hasEndpoint()) continue; - controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); + DeploymentId deployment = new DeploymentId(application.id(), zone); + controller().routing().of(deployment).configure(DeploymentSpec.empty); } } return 1.0; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index e8a7f7729fb..4b060846090 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -31,9 +31,9 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import java.security.PublicKey; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5cd5a70e4a4..1046737d860 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -44,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; @@ -97,10 +96,11 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; @@ -1292,18 +1292,22 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { request.getUri()).toString()); } } - // Add dummy values for not-yet-existent prod deployments. - status.jobSteps().keySet().stream() - .filter(job -> job.application().instance().equals(instance.name())) - .filter(job -> job.type().isProduction() && job.type().isDeployment()) + // Add dummy values for not-yet-existent prod deployments, and running dev/perf deployments. + Stream.concat(status.jobSteps().keySet().stream() + .filter(job -> job.application().instance().equals(instance.name())) + .filter(job -> job.type().isProduction() && job.type().isDeployment()), + controller.jobController().active(instance.id()).stream() + .map(run -> run.id().job()) + .filter(job -> job.type().environment().isManuallyDeployed())) .map(job -> job.type().zone(controller.system())) .filter(zone -> ! instance.deployments().containsKey(zone)) .forEach(zone -> { - Cursor deploymentObject = instancesArray.addObject(); - deploymentObject.setString("environment", zone.environment().value()); - deploymentObject.setString("region", zone.region().value()); + Cursor deploymentObject = instancesArray.addObject(); + deploymentObject.setString("environment", zone.environment().value()); + deploymentObject.setString("region", zone.region().value()); }); + // TODO jonmv: Remove when clients are updated application.deployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", KeyUtils.toPem(key))); @@ -1532,49 +1536,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (deployment == null) { throw new NotExistsException(instance + " has no deployment in " + zone); } - - // The order here matters because setGlobalRotationStatus involves an external request that may fail. - // TODO(mpolden): Set only one of these when only one kind of global endpoints are supported per zone. - var deploymentId = new DeploymentId(instance.id(), zone); - setGlobalRotationStatus(deploymentId, inService, request); - setGlobalEndpointStatus(deploymentId, inService, request); - + DeploymentId deploymentId = new DeploymentId(instance.id(), zone); + RoutingStatus.Agent agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; + RoutingStatus.Value status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out; + controller.routing().of(deploymentId).setRoutingStatus(status, agent); return new MessageResponse(Text.format("Successfully set %s in %s %s service", instance.id().toShortString(), zone, inService ? "in" : "out of")); } - /** Set the global endpoint status for given deployment. This only applies to global endpoints backed by a cloud service */ - private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { - var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; - var status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out; - controller.routing().policies().setRoutingStatus(deployment, status, agent); - } - - /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ - private void setGlobalRotationStatus(DeploymentId deployment, boolean inService, HttpRequest request) { - var requestData = toSlime(request.getData()).get(); - var reason = mandatory("reason", requestData).asString(); - var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; - long timestamp = controller.clock().instant().getEpochSecond(); - var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; - var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp); - controller.routing().setGlobalRotationStatus(deployment, endpointStatus); - } - private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); - controller.routing().globalRotationStatus(deploymentId) - .forEach((endpoint, status) -> { - array.addString(endpoint.upstreamIdOf(deploymentId)); - Cursor statusObject = array.addObject(); - statusObject.setString("status", status.getStatus().name()); - statusObject.setString("reason", status.getReason() == null ? "" : status.getReason()); - statusObject.setString("agent", status.getAgent() == null ? "" : status.getAgent()); - statusObject.setLong("timestamp", status.getEpoch()); - }); + Optional<Endpoint> primaryEndpoint = controller.routing().readDeclaredEndpointsOf(deploymentId.applicationId()) + .requiresRotation() + .primary(); + if (primaryEndpoint.isPresent()) { + DeploymentRoutingContext context = controller.routing().of(deploymentId); + RoutingStatus status = context.routingStatus(); + array.addString(primaryEndpoint.get().upstreamIdOf(deploymentId)); + Cursor statusObject = array.addObject(); + statusObject.setString("status", status.value().name()); + statusObject.setString("reason", ""); + statusObject.setString("agent", status.agent().name()); + statusObject.setLong("timestamp", status.changedAt().getEpochSecond()); + } return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 45abf7f2946..226a7ca9561 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -19,26 +19,26 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; import com.yahoo.yolean.Exceptions; import java.net.URI; -import java.time.Instant; import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.logging.Level; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * This implements the /routing/v1 API, which provides operators and tenants routing control at both zone- (operator @@ -112,11 +112,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var deploymentsStatus = deployments.stream() .collect(Collectors.toMap( deploymentId -> deploymentId, - deploymentId -> Stream.concat( - directGlobalRoutingStatus(deploymentId).stream(), - sharedGlobalRoutingStatus(deploymentId).stream() - ).collect(Collectors.toList()) - )); + deploymentId -> controller.routing().of(deploymentId).routingStatus()) + ); var slime = new Slime(); var root = slime.setObject(); @@ -125,11 +122,11 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var endpointRoot = endpointsRoot.addObject(); endpointToSlime(endpointRoot, endpoint); var zonesRoot = endpointRoot.setArray("zones"); - endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value())).forEach(deployment -> { - deploymentsStatus.getOrDefault(deployment, List.of()).forEach(status -> { - deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod()); - }); - }); + endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value())) + .forEach(deployment -> { + RoutingStatus status = deploymentsStatus.get(deployment); + deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod()); + }); }); return new SlimeJsonResponse(slime); @@ -211,13 +208,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private HttpResponse setZoneStatus(Path path, boolean in) { - var zone = zoneFrom(path); - if (exclusiveRoutingIn(zone)) { - var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; - controller.routing().policies().setRoutingStatus(zone, status); - } else { - controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); - } + ZoneId zone = zoneFrom(path); + RoutingContext context = controller.routing().of(zone); + RoutingStatus.Value newStatus = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; + context.setRoutingStatus(newStatus, RoutingStatus.Agent.operator); return new MessageResponse("Set global routing status for deployments in " + zone + " to " + (in ? "IN" : "OUT")); } @@ -231,16 +225,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private void toSlime(ZoneId zone, Cursor zoneObject) { - if (exclusiveRoutingIn(zone)) { - var zonePolicy = controller.routing().policies().get(zone); - zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.routingStatus(), RoutingMethod.exclusive); - } else { - // Rotation status per zone only exposes in/out status, no agent or time of change. - var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); - var globalRouting = new RoutingStatus(in ? RoutingStatus.Value.in : RoutingStatus.Value.out, - RoutingStatus.Agent.operator, Instant.EPOCH); - zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared); - } + RoutingContext context = controller.routing().of(zone); + zoneStatusToSlime(zoneObject, zone, context.routingStatus(), context.routingMethod()); } private HttpResponse setDeploymentStatus(Path path, boolean in, HttpRequest request) { @@ -249,18 +235,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; requireDeployment(deployment, instance); - - if (sharedRoutingIn(deployment.zoneId())) { - // Set rotation status - var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, - "", - agent.name(), - controller.clock().instant().getEpochSecond()); - controller.routing().setGlobalRotationStatus(deployment, endpointStatus); - } else { - // Set policy status - controller.routing().policies().setRoutingStatus(deployment, status, agent); - } + controller.routing().of(deployment).setRoutingStatus(status, agent); return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } @@ -279,66 +254,24 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var instances = instanceId == null ? application.instances().values() : List.of(application.instances().get(instanceId.instance())); + EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application); for (var instance : instances) { var zones = zoneId == null ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)) .collect(Collectors.toList()) : List.of(zoneId); for (var zone : zones) { - var deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance); - // Include status from rotation - sharedGlobalRoutingStatus(deploymentId).ifPresent(status -> { - deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.shared); - }); - - // Include status from routing policies - directGlobalRoutingStatus(deploymentId).forEach(status -> { - deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.exclusive); - }); - } - } - } - - } - - private Optional<RoutingStatus> sharedGlobalRoutingStatus(DeploymentId deploymentId) { - if (sharedRoutingIn(deploymentId.zoneId())) { - var rotationStatus = controller.routing().globalRotationStatus(deploymentId); - // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. - var endpointStatus = rotationStatus.values().stream().findFirst(); - if (endpointStatus.isPresent()) { - var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); - RoutingStatus.Agent agent; - try { - agent = RoutingStatus.Agent.valueOf(endpointStatus.get().getAgent()); - } catch (IllegalArgumentException e) { - agent = RoutingStatus.Agent.unknown; + DeploymentId deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance); + DeploymentRoutingContext context = controller.routing().of(deploymentId); + if (declaredEndpoints.targets(deploymentId).isEmpty()) continue; // No declared endpoints point to this deployment + deploymentStatusToSlime(deploymentsArray.addObject(), + deploymentId, + context.routingStatus(), + context.routingMethod()); } - var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in - ? RoutingStatus.Value.in - : RoutingStatus.Value.out; - return Optional.of(new RoutingStatus(status, agent, changedAt)); } } - return Optional.empty(); - } - - private List<RoutingStatus> directGlobalRoutingStatus(DeploymentId deploymentId) { - return controller.routing().policies().get(deploymentId).values().stream() - .filter(p -> ! p.instanceEndpoints().isEmpty()) // This policy does not apply to a global endpoint - .filter(p -> exclusiveRoutingIn(p.id().zone())) - .map(p -> p.status().routingStatus()) - .collect(Collectors.toList()); - } - - /** Returns whether given zone uses exclusive routing */ - private boolean exclusiveRoutingIn(ZoneId zone) { - return controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive); - } - /** Returns whether given zone uses shared routing */ - private boolean sharedRoutingIn(ZoneId zone) { - return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); } private static void zoneStatusToSlime(Cursor object, ZoneId zone, RoutingStatus routingStatus, RoutingMethod method) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 2a39ed08014..d2dc2771160 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -215,7 +215,7 @@ public class RoutingPolicies { Application application = controller.applications().requireApplication(routingTable.keySet().iterator().next().application()); Map<DeploymentId, Map<EndpointId, Integer>> targetWeights = targetWeights(application); - Map<String, Set<AliasTarget>> targetsByEndpoint = new LinkedHashMap<>(); + Map<Endpoint, Set<AliasTarget>> targetsByEndpoint = new LinkedHashMap<>(); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); EndpointList endpoints = controller.routing().declaredEndpointsOf(application) @@ -230,19 +230,27 @@ public class RoutingPolicies { for (var policy : routeEntry.getValue()) { for (var target : endpoint.targets()) { if (!policy.appliesTo(target.deployment())) continue; + if (policy.dnsZone().isEmpty()) continue; // Does not support ALIAS records int weight = target.weight(); if (isConfiguredOut(policy, inactiveZones) && removableFromApplicationEndpoint(policy, application, targetWeights)) { weight = 0; } WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), target.deployment().zoneId(), weight); - targetsByEndpoint.computeIfAbsent(endpoint.dnsName(), (k) -> new LinkedHashSet<>()) + targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()) .add(weightedAliasTarget); } } } targetsByEndpoint.forEach((applicationEndpoint, targets) -> { - controller.nameServiceForwarder().createAlias(RecordName.from(applicationEndpoint), targets, Priority.normal); + ZoneId targetZone = applicationEndpoint.targets().stream() + .map(Endpoint.Target::deployment) + .map(DeploymentId::zoneId) + .findFirst() + .get(); + nameServiceForwarderIn(targetZone).createAlias(RecordName.from(applicationEndpoint.dnsName()), + targets, + Priority.normal); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java new file mode 100644 index 00000000000..28fbeee28f5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -0,0 +1,154 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.LockedApplication; +import com.yahoo.vespa.hosted.controller.RoutingController; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment. + * + * @author mpolden + */ +public abstract class DeploymentRoutingContext implements RoutingContext { + + final DeploymentId deployment; + final RoutingController controller; + final RoutingMethod method; + + public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController controller) { + this.deployment = Objects.requireNonNull(deployment); + this.controller = Objects.requireNonNull(controller); + this.method = Objects.requireNonNull(method); + } + + /** + * Prepare routing configuration for the deployment in this context + * + * @return the container endpoints relevant for this deployment, as declared in deployment spec + */ + public final Set<ContainerEndpoint> prepare(LockedApplication application) { + return controller.containerEndpointsOf(application, deployment.applicationId().instance(), deployment.zoneId()); + } + + /** Configure routing for the deployment in this context, using given deployment spec */ + public final void configure(DeploymentSpec deploymentSpec) { + controller.policies().refresh(deployment.applicationId(), deploymentSpec, deployment.zoneId()); + } + + /** Routing method of this context */ + public final RoutingMethod routingMethod() { + return method; + } + + /** Read the routing policy for given cluster in this deployment */ + public final Optional<RoutingPolicy> routingPolicy(ClusterSpec.Id cluster) { + RoutingPolicyId id = new RoutingPolicyId(deployment.applicationId(), cluster, deployment.zoneId()); + return Optional.ofNullable(controller.policies().get(deployment).get(id)); + } + + /** + * Extension of a {@link DeploymentRoutingContext} for deployments using either {@link RoutingMethod#shared} or + * {@link RoutingMethod#sharedLayer4} routing. + */ + public static class SharedDeploymentRoutingContext extends DeploymentRoutingContext { + + private final Clock clock; + private final ConfigServer configServer; + + public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) { + super(deployment, RoutingMethod.shared, controller); + this.clock = Objects.requireNonNull(clock); + this.configServer = Objects.requireNonNull(configServer); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in + ? EndpointStatus.Status.in + : EndpointStatus.Status.out, + "", + agent.name(), + clock.instant().getEpochSecond()); + primaryEndpoint().ifPresent(endpoint -> { + try { + configServer.setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), newStatus); + } catch (Exception e) { + throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); + } + }); + } + + @Override + public RoutingStatus routingStatus() { + Optional<EndpointStatus> status = primaryEndpoint().map(endpoint -> { + var upstreamName = endpoint.upstreamIdOf(deployment); + return configServer.getGlobalRotationStatus(deployment, upstreamName); + }); + if (status.isEmpty()) return RoutingStatus.DEFAULT; + RoutingStatus.Agent agent; + try { + agent = RoutingStatus.Agent.valueOf(status.get().getAgent().toLowerCase()); + } catch (IllegalArgumentException e) { + agent = RoutingStatus.Agent.unknown; + } + return new RoutingStatus(status.get().getStatus() == EndpointStatus.Status.in + ? RoutingStatus.Value.in + : RoutingStatus.Value.out, + agent, + Instant.ofEpochSecond(status.get().getEpoch())); + } + + private Optional<Endpoint> primaryEndpoint() { + return controller.readDeclaredEndpointsOf(deployment.applicationId()) + .requiresRotation() + .primary(); + } + + } + + /** + * Implementation of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#exclusive} + * routing. + */ + public static class ExclusiveDeploymentRoutingContext extends DeploymentRoutingContext { + + public ExclusiveDeploymentRoutingContext(DeploymentId deployment, RoutingController controller) { + super(deployment, RoutingMethod.exclusive, controller); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + controller.policies().setRoutingStatus(deployment, value, agent); + } + + @Override + public RoutingStatus routingStatus() { + // Status for a deployment applies to all clusters within the deployment, so we use the status from the + // first matching policy here + return controller.policies().get(deployment).values().stream() + .findFirst() + .map(RoutingPolicy::status) + .map(RoutingPolicy.Status::routingStatus) + .orElse(RoutingStatus.DEFAULT); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java new file mode 100644 index 00000000000..e949c45f2fd --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +import java.util.Objects; + +/** + * An implementation of {@link RoutingContext} for a zone using {@link RoutingMethod#exclusive} routing. + * + * @author mpolden + */ +public class ExclusiveRoutingContext implements RoutingContext { + + private final RoutingPolicies policies; + private final ZoneId zone; + + public ExclusiveRoutingContext(ZoneId zone, RoutingPolicies policies) { + this.policies = Objects.requireNonNull(policies); + this.zone = Objects.requireNonNull(zone); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + policies.setRoutingStatus(zone, value); + } + + @Override + public RoutingStatus routingStatus() { + return policies.get(zone).routingStatus(); + } + + @Override + public RoutingMethod routingMethod() { + return RoutingMethod.exclusive; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java new file mode 100644 index 00000000000..6f43416b9b5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +/** + * Top-level interface for a routing context, which provides control of routing status for a deployment or zone. + * + * @author mpolden + */ +public interface RoutingContext { + + /** Change the routing status for the zone or deployment represented by this context */ + void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent); + + /** Get the current routing status for the zone or deployment represented by this context */ + RoutingStatus routingStatus(); + + /** Routing method used in this context */ + RoutingMethod routingMethod(); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java new file mode 100644 index 00000000000..e38212d7f80 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +import java.time.Instant; +import java.util.Objects; + +/** + * An implementation of {@link RoutingContext} for a zone, using either {@link RoutingMethod#shared} or + * {@link RoutingMethod#sharedLayer4} routing. + * + * @author mpolden + */ +public class SharedRoutingContext implements RoutingContext { + + private final ConfigServer configServer; + private final ZoneId zone; + + public SharedRoutingContext(ZoneId zone, ConfigServer configServer) { + this.configServer = Objects.requireNonNull(configServer); + this.zone = Objects.requireNonNull(zone); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + boolean in = value == RoutingStatus.Value.in; + configServer.setGlobalRotationStatus(zone, in); + } + + @Override + public RoutingStatus routingStatus() { + boolean in = configServer.getGlobalRotationStatus(zone); + RoutingStatus.Value newValue = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; + return new RoutingStatus(newValue, + RoutingStatus.Agent.operator, + Instant.EPOCH); // API does not support time of change + } + + @Override + public RoutingMethod routingMethod() { + return RoutingMethod.shared; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java index ca5d2d5915f..0cf7101cac0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.text.Text; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java index 2b75777fbbd..4d97962a40a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java index fe9280b1193..36a43f80e9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.vespa.curator.Lock; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java index 5b24f39717b..961fdc6dd9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java index 032f01433b3..19e816a0b51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; /** * The possible states of a global rotation. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java index 1ddbd640e53..6d95ad9a230 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Deployment; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 9472801ef2c..30cdd1b8466 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -38,8 +37,10 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; @@ -214,22 +215,18 @@ public class ControllerTest { // Check initial rotation status var deployment1 = context.deploymentIdIn(zone1); - var status1 = tester.controller().routing().globalRotationStatus(deployment1); - assertEquals(1, status1.size()); - assertTrue("All upstreams are in", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); + DeploymentRoutingContext routingContext = tester.controller().routing().of(deployment1); + RoutingStatus status1 = routingContext.routingStatus(); + assertEquals(RoutingStatus.Value.in, status1.value()); // Set the deployment out of service in the global rotation - var newStatus = new EndpointStatus(EndpointStatus.Status.out, "unit-test", ControllerTest.class.getSimpleName(), tester.clock().instant().getEpochSecond()); - tester.controller().routing().setGlobalRotationStatus(deployment1, newStatus); - status1 = tester.controller().routing().globalRotationStatus(deployment1); - assertEquals(1, status1.size()); - assertTrue("All upstreams are out", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); - assertTrue("Reason is set", status1.values().stream().allMatch(es -> es.getReason().equals("unit-test"))); + routingContext.setRoutingStatus(RoutingStatus.Value.out, RoutingStatus.Agent.operator); + RoutingStatus status2 = routingContext.routingStatus(); + assertEquals(RoutingStatus.Value.out, status2.value()); // Other deployment remains in - var status2 = tester.controller().routing().globalRotationStatus(context.deploymentIdIn(zone2)); - assertEquals(1, status2.size()); - assertTrue("All upstreams are in", status2.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); + RoutingStatus status3 = tester.controller().routing().of(context.deploymentIdIn(zone2)).routingStatus(); + assertEquals(RoutingStatus.Value.in, status3.value()); } @Test @@ -625,13 +622,13 @@ public class ControllerTest { .instances("beta,main") .region("us-west-1") .region("us-east-3") - .applicationEndpoint("a", "qrs", "us-west-1", + .applicationEndpoint("a", "default", "us-west-1", Map.of(InstanceName.from("beta"), 2, InstanceName.from("main"), 8)) - .applicationEndpoint("b", "qrs", "us-west-1", + .applicationEndpoint("b", "default", "us-west-1", Map.of(InstanceName.from("beta"), 1, InstanceName.from("main"), 1)) - .applicationEndpoint("c", "qrs", "us-east-3", + .applicationEndpoint("c", "default", "us-east-3", Map.of(InstanceName.from("beta"), 4, InstanceName.from("main"), 6)) .build(); @@ -640,11 +637,11 @@ public class ControllerTest { // Endpoint names are passed to each deployment DeploymentId usWest = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); DeploymentId usEast = context.deploymentIdIn(ZoneId.from("prod", "us-east-3")); - Map<DeploymentId, List<String>> deploymentEndpoints = Map.of(usWest, List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", "b--app1--tenant1.us-west-1-r.vespa.oath.cloud"), - usEast, List.of("c--app1--tenant1.us-east-3-r.vespa.oath.cloud")); + Map<DeploymentId, List<String>> deploymentEndpoints = Map.of(usWest, List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", "b.app1.tenant1.us-west-1-r.vespa.oath.cloud"), + usEast, List.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud")); deploymentEndpoints.forEach((zone, endpointNames) -> { assertEquals("Endpoint names are passed to config server in " + zone, - Set.of(new ContainerEndpoint("qrs", "application", + Set.of(new ContainerEndpoint("default", "application", endpointNames)), tester.configServer().containerEndpoints().get(zone)); }); @@ -653,21 +650,21 @@ public class ControllerTest { // DNS records are created for each endpoint Set<Record> records = tester.controllerTester().nameService().records(); assertEquals(Set.of(new Record(Record.Type.CNAME, - RecordName.from("a--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + RecordName.from("a.app1.tenant1.us-west-1-r.vespa.oath.cloud"), RecordData.from("vip.prod.us-west-1.")), new Record(Record.Type.CNAME, - RecordName.from("b--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + RecordName.from("b.app1.tenant1.us-west-1-r.vespa.oath.cloud"), RecordData.from("vip.prod.us-west-1.")), new Record(Record.Type.CNAME, - RecordName.from("c--app1--tenant1.us-east-3-r.vespa.oath.cloud"), + RecordName.from("c.app1.tenant1.us-east-3-r.vespa.oath.cloud"), RecordData.from("vip.prod.us-east-3."))), records); List<String> endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application()) .scope(Endpoint.Scope.application) .mapToList(Endpoint::dnsName); - assertEquals(List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", - "b--app1--tenant1.us-west-1-r.vespa.oath.cloud", - "c--app1--tenant1.us-east-3-r.vespa.oath.cloud"), + assertEquals(List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", + "b.app1.tenant1.us-west-1-r.vespa.oath.cloud", + "c.app1.tenant1.us-east-3-r.vespa.oath.cloud"), endpointDnsNames); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 11086ff7663..5cf554f2c01 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -521,22 +521,16 @@ public class InternalStepRunnerTest { } @Test - public void generates_correct_services_xml_using_osgi_based_runtime() { - generates_correct_services_xml("test_runner_services.xml-cd-osgi", true); + public void generates_correct_services_xml() { + generates_correct_services_xml("test_runner_services.xml-cd"); } - @Test - public void generates_correct_services_xml_using_legacy_runtime() { - generates_correct_services_xml("test_runner_services.xml-cd-legacy", false); - } - - private void generates_correct_services_xml(String filenameExpectedOutput, boolean useOsgiBasedRuntime) { + private void generates_correct_services_xml(String filenameExpectedOutput) { ControllerConfig.Steprunner.Testerapp config = new ControllerConfig.Steprunner.Testerapp.Builder().build(); assertFile(filenameExpectedOutput, new String(InternalStepRunner.servicesXml( true, false, - useOsgiBasedRuntime, new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local), config))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index f1421b5affd..b33f8f6f7e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -23,9 +23,9 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import org.junit.Test; import java.nio.file.Files; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index ae6232ae419..afd67824ee8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -69,6 +69,7 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; @@ -1868,13 +1869,12 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void assertGlobalRouting(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) { - var changedAt = tester.controller().clock().instant(); - var westPolicies = tester.controller().routing().policies().get(deployment); - assertEquals(1, westPolicies.size()); - var westPolicy = westPolicies.values().iterator().next(); - assertEquals(value, westPolicy.status().routingStatus().value()); - assertEquals(agent, westPolicy.status().routingStatus().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().routingStatus().changedAt()); + Instant changedAt = tester.controller().clock().instant(); + DeploymentRoutingContext context = tester.controller().routing().of(deployment); + RoutingStatus status = context.routingStatus(); + assertEquals(value, status.value()); + assertEquals(agent, status.agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.SECONDS), status.changedAt()); } private static class RequestBuilder implements Supplier<Request> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index fb6088f54b8..ab2a3bf945c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -24,9 +24,9 @@ { "cluster": "foo", "tls": true, - "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/", + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", "scope": "application", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json index 934e0cf43b9..de2266fd197 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json @@ -4,7 +4,7 @@ { "status": "in", "reason": "", - "agent": "", + "agent": "unknown", "timestamp": 1497618757 } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 409e97b063c..62ad3a2db7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -27,9 +27,9 @@ { "cluster": "foo", "tls": true, - "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/", + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", "scope": "application", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index e7c2eacbd02..9a3ac8b547d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; +package com.yahoo.vespa.hosted.controller.routing.rotation; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi b/controller-server/src/test/resources/test_runner_services.xml-cd index 634137e3fb6..634137e3fb6 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi +++ b/controller-server/src/test/resources/test_runner_services.xml-cd diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-legacy b/controller-server/src/test/resources/test_runner_services.xml-cd-legacy deleted file mode 100644 index c6046479934..00000000000 --- a/controller-server/src/test/resources/test_runner_services.xml-cd-legacy +++ /dev/null @@ -1,22 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<services xmlns:deploy='vespa' version='1.0'> - <container version='1.0' id='tester'> - - <component id="com.yahoo.vespa.hosted.testrunner.TestRunner" bundle="vespa-testrunner-components"> - <config name="com.yahoo.vespa.hosted.testrunner.test-runner"> - <artifactsPath>artifacts</artifactsPath> - <surefireMemoryMb>5120</surefireMemoryMb> - <useAthenzCredentials>true</useAthenzCredentials> - <useTesterCertificate>false</useTesterCertificate> - </config> - </component> - - <handler id="com.yahoo.vespa.testrunner.TestRunnerHandler" bundle="vespa-osgi-testrunner"> - <binding>http://*/tester/v1/*</binding> - </handler> - - <nodes count="1" allocated-memory="17%"> - <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/> - </nodes> - </container> -</services> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index e482439dd7d..b0dfed2bfd5 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -34,7 +34,7 @@ function(setup_vespa_default_build_settings_centos_8) if (VESPA_OS_DISTRO_NAME STREQUAL "CentOS Stream") set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) else() - set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) endif() endfunction() diff --git a/dist/vespa.spec b/dist/vespa.spec index 031213ac693..bfb4c14bdeb 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -101,7 +101,7 @@ BuildRequires: libarchive %if 0%{?_centos_stream} BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) %else -BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12) +BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) %endif %else BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11) @@ -246,7 +246,7 @@ Requires: vespa-gtest = 1.11.0 %if 0%{?_centos_stream} %define _vespa_llvm_version 12 %else -%define _vespa_llvm_version 11 +%define _vespa_llvm_version 12 %endif %else %define _vespa_llvm_version 10 @@ -375,7 +375,7 @@ Requires: openssl-libs %if 0%{?_centos_stream} Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) %else -Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12) +Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) %endif %else Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 6260ce07c3f..685d2a351c5 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -384,6 +384,20 @@ public class Flags { "Takes effect on config server restart", ZONE_ID); + public static final UnboundBooleanFlag USE_V8_GEO_POSITIONS = defineFeatureFlag( + "use-v8-geo-positions", false, + List.of("arnej"), "2021-11-15", "2022-12-31", + "Use Vespa 8 types and formats for geographical positions", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag USE_LEGACY_LB_SERVICES = defineFeatureFlag( + "use-legacy-lb-services", true, + List.of("tokle"), "2021-11-22", "2021-12-31", + "Whether to generate routing table based on legacy lb-services config", + "Takes effect on container reboot", + ZONE_ID, HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 593c5e5f05b..34547d14616 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -114,7 +114,7 @@ public class PermanentFlags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag( - "zookeeper-server-version", "3.6.3", + "zookeeper-server-version", "3.7.0", "ZooKeeper server version, a jar file zookeeper-server-<ZOOKEEPER_SERVER_VERSION>-jar-with-dependencies.jar must exist", "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); diff --git a/sd-plugin/.gitignore b/integration/intellij/.gitignore index fa33885331d..96a25be6725 100644 --- a/sd-plugin/.gitignore +++ b/integration/intellij/.gitignore @@ -12,4 +12,9 @@ gradle-app.setting !gradle/wrapper/gradle-wrapper.jar # Cache of project -.gradletasknamecache
\ No newline at end of file +.gradletasknamecache + +# These are not needed, but IntelliJ automatically adds them +gradlew +gradlew.bat +gradle/ diff --git a/integration/intellij/BACKLOG.md b/integration/intellij/BACKLOG.md new file mode 100644 index 00000000000..aaef8ddd411 --- /dev/null +++ b/integration/intellij/BACKLOG.md @@ -0,0 +1,41 @@ +### Open Issues + +In some cases, the parser stops on bad syntax and can't build the PSI tree. +That means no features will work in the file. + +To enable the grammar recognize some keywords as identifiers (e.g. "filter" as a field's name), +the identifier rule (named "IdentifierVal") wraps the regex (ID_REG) and the KeywordOrIdentifier +rule (which contains all the keywords in the language). + +The implementation of the GoTo Declaration feature is not exactly the same as IntelliJ. +In IntelliJ if a reference has several declarations, after clicking "Goto Declaration" +there is a little window with all the declarations to choose from. +It can be done by changing the method "multiResolve" in SdReference.java to return +more than one declaration. The problem with that is that it causes the "Find Usages" +feature to not work. For now the plugin "Goto Declaration" feature shows only the +most specific declaration by the right rank-profile scope. + +The "Find Usages" window can group usages only under rank-profiles and document-summaries. +Other usages appear directly under the .sd file. To create another group type of usages' group, +you'll need to create 2 classes: one for the extension "fileStructureGroupRuleProvider" +(e.g. SdRankProfileGroupingRuleProvider.java), and one for the +grouping rule itself (e.g. SdRankProfileGroupingRule.java). +Another open problem is that the navigation isn't working in the current grouping rules. +It means that when clicking on the group headline (e.g. some name of a rank-profile) +the IDE doesn't "jump" to the matching declaration. + +Goto declaration doesn't work for document and schema inherits. E.g. if document A inherits from +document B, B doesn't have a reference to its declaration. + +There aren't any tests for the plugin. + +Semicolons in indexing statements are marked with red background for some reason. +They are not marked as errors. + +Type suggestions should include all primitive types, not just annotations + +Even if the parser continues, only the first error in a file is marked. + +Aliases with dot does not work. + + diff --git a/integration/intellij/README.md b/integration/intellij/README.md new file mode 100644 index 00000000000..fec547f8a06 --- /dev/null +++ b/integration/intellij/README.md @@ -0,0 +1,52 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +# SD Reader + +IntelliJ plugin for working with Vespa application packages. + +## Using the plugin + +Download it from JetBrains Marketplace. + +## Using a local build + +Build (see below) and load it in IntelliJ by choosing +Preferences -> Plugins -> Press the gear icon -> Install Plugin from Disk. + +## Building the plugin + + gradle + +This produces an installable plugin .zip in the directory build/distributions + +*Prerequisite*: gradle 7. + +Why gradle? Because it's what JetBrains supports for building plugins. +However, gradle is configured with a maven directory layout. + +## Optional IntelliJ plugins for working with plugin development + +1. Plugin DevKit +2. Grammar-Kit: For reading the .bnf file. +3. PsiViewer: Helps testing the bnf grammar. + +With the first (?), you can run the gradle task "intellij/runIde" (or "./gradlew runIde" in the command line), +open a project with some sd file and see how the plugin works on it. + + +## Some useful links: + +1. JetBrains official tutorials: https://plugins.jetbrains.com/docs/intellij/custom-language-support.html and + https://plugins.jetbrains.com/docs/intellij/custom-language-support-tutorial.html + +2. Grammar-Kit HOWTO: Helps to understand the BNF syntax. + https://github.com/JetBrains/Grammar-Kit/blob/master/HOWTO.md + +3. How to deal with left-recursion in the grammar (in SD for example it happens in expressions). Last answer here: + https://intellij-support.jetbrains.com/hc/en-us/community/posts/360001258300-What-s-the-alternative-to-left-recursion-in-GrammarKit- + +4. Great tutorial for a custom-language-plugin, but only for the basics (mainly the parser and lexer): + https://medium.com/@shan1024/custom-language-plugin-development-for-intellij-idea-part-01-d6a41ab96bc9 + +5. Code of Dart (some custom language) plugin for IntelliJ: + https://github.com/JetBrains/intellij-plugins/tree/0f07ca63355d5530b441ca566c98f17c560e77f8/Dart
\ No newline at end of file diff --git a/sd-plugin/build.gradle b/integration/intellij/build.gradle index e5b39769b96..4801bc810b0 100644 --- a/sd-plugin/build.gradle +++ b/integration/intellij/build.gradle @@ -1,5 +1,4 @@ -// This "noinspection" comment below is here to fix a warning -//noinspection GroovyAssignabilityCheck +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. plugins { id 'org.jetbrains.intellij' version '1.1.4' id 'java' @@ -17,17 +16,17 @@ import org.jetbrains.grammarkit.tasks.GenerateLexer import org.jetbrains.grammarkit.tasks.GenerateParser task generateSdLexer(type: GenerateLexer) { - source 'src/main/java/org/intellij/sdk/language/lexer/sd.flex' - targetDir 'src/main/gen/org/intellij/sdk/language/lexer/' + source 'src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex' + targetDir 'target/generated-sources/jflex/ai/vespa/intellij/schema/lexer/' targetClass 'SdLexer' purgeOldFiles true } task generateSdParser(type: GenerateParser) { - source 'src/main/java/org/intellij/sdk/language/parser/sd.bnf' - targetRoot 'src/main/gen' - pathToParser 'org/intellij/sdk/language/parser/SdParser.java' - pathToPsiRoot 'org/intellij/sdk/language/psi/' + source 'src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf' + targetRoot 'target/generated-sources/bnf/' + pathToParser 'ai/vespa/intellij/schema/parser/SdParser.java' + pathToPsiRoot 'ai/vespa/intellij/schema/parser/psi/' purgeOldFiles true } @@ -36,20 +35,18 @@ compileJava { dependsOn generateSdParser } - -group 'org.yahoo.native' -version '1.0.2' +group 'ai.vespa' +version '1.0.0' // Also update pom.xml version if this is changed sourceCompatibility = 11 // This "noinspection" comment below is here to fix a warning -//noinspection GroovyAssignabilityCheck +// noinspection GroovyAssignabilityCheck repositories { mavenCentral() } - -sourceSets.main.java.srcDirs 'src/main/gen' +sourceSets.main.java.srcDirs = ['src/main/java', 'target/generated-sources/bnf', 'target/generated-sources/jflex'] // See https://github.com/JetBrains/gradle-intellij-plugin/ intellij { diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml new file mode 100644 index 00000000000..84a24b788b8 --- /dev/null +++ b/integration/intellij/pom.xml @@ -0,0 +1,90 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle --> + <version>1.0.0</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle --> + <description> + Maven wrapper for the gradle build of this IntelliJ plugin. + </description> + + <dependencies> + </dependencies> + + <build> + <plugins> + <!-- The Gradle plugin must be built by Gradle; therefore the compilation is skipped. --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <skipMain>true</skipMain> + <skip>true</skip> + </configuration> + </plugin> + <!-- Gradle is also responsible for creating javadoc such that this task is skipped here. --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + <!-- Tie Maven executions into the Gradle life-cycle. --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>gradle-clean</id> + <phase>clean</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>gradle</executable> + <arguments> + <argument>clean</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>gradle-build</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>gradle</executable> + </configuration> + </execution> + </executions> + </plugin> + <!-- Copies the artifact created by Gradle back to the Maven target folder. --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>copy-zip</id> + <phase>install</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <copy file="build/distributions/vespa-${project.version}.zip" todir="target" overwrite="true" /> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/integration/intellij/settings.gradle b/integration/intellij/settings.gradle new file mode 100644 index 00000000000..94afd49df08 --- /dev/null +++ b/integration/intellij/settings.gradle @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +// This sets the name of the generated zip file uploadable to Intellij Marketplace +// (But not the actual *name* of the plugin, which is the name in plugin.xml) +rootProject.name = 'vespa' diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf index 67bdca7d2dc..6be7302a28c 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf +++ b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf @@ -1,11 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** -This file is the SD grammar. -NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many) -@author: Shahar Ariel + * Vespa schema file grammar. + * NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many) + * + * @author: Shahar Ariel */ { - parserClass="org.intellij.sdk.language.parser.SdParser" // Name and the location of the parser which will be generated. + parserClass="ai.vespa.intellij.schema.parser.SdParser" // Name and the location of the parser which will be generated. extends="com.intellij.extapi.psi.ASTWrapperPsiElement" // All nodes will extend this class. Wraps AST node to a PSI node. @@ -13,13 +15,13 @@ NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats psiClassPrefix="Sd" psiImplClassSuffix="Impl" - psiPackage="org.intellij.sdk.language.psi" // Location to be used when generating PSI classes. - psiImplPackage="org.intellij.sdk.language.psi.impl" // Location to be used when generating PSI implementation classes. + psiPackage="ai.vespa.intellij.schema.psi" // Location to be used when generating PSI classes. + psiImplPackage="ai.vespa.intellij.schema.psi.impl" // Location to be used when generating PSI implementation classes. - elementTypeHolderClass="org.intellij.sdk.language.psi.SdTypes" // Element type holder class name. + elementTypeHolderClass="ai.vespa.intellij.schema.psi.SdTypes" // Element type holder class name. - elementTypeClass="org.intellij.sdk.language.psi.SdElementType" // Class which will be used to create internal nodes. - tokenTypeClass="org.intellij.sdk.language.psi.SdTokenType" // Class which will be used to create leaf nodes. + elementTypeClass="ai.vespa.intellij.schema.psi.SdElementType" // Class which will be used to create internal nodes. + tokenTypeClass="ai.vespa.intellij.schema.psi.SdTokenType" // Class which will be used to create leaf nodes. extends(".*Expr")=RankingExpression // Here to deal with left-recursion that happens in expressions @@ -30,7 +32,7 @@ NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats COMMENT = 'regexp:#.*' SYMBOL = 'regexp:[!$|:{}(),.\[\]]' COMPARISON_OPERATOR = 'regexp:[<>]|(==)|(<=)|(>=)|(~=)' - ARITHMETIC_OPERATOR = 'regexp:[\-+*/]' + ARITHMETIC_OPERATOR = 'regexp:[\-+*/%]' INTEGER_REG = 'regexp:[0-9]+' FLOAT_REG = 'regexp:[0-9]+[.][0-9]+[e]?' STRING_REG = 'regexp:\"([^\"\\]*(\\.[^\"\\]*)*)\"' @@ -42,14 +44,14 @@ SdFile ::= SchemaDefinition | DocumentDefinition SchemaDefinition ::= (search | schema) IdentifierVal? (inherits IdentifierVal)? '{' SchemaBody '}' SchemaBody ::= SchemaBodyOptions* DocumentDefinition SchemaBodyOptions* // Does not support zero-or-one occurrences private SchemaBodyOptions ::= SchemaFieldDefinition | ImportFieldDefinition | DocumentSummaryDefinition | - RankProfileDefinition | IndexDefinition | + RankProfileDefinition | IndexDefinition | DocumentStructDefinition | FieldSetDefinition | ConstantDefinition | OnnxModelDefinition | StemmingDefinition | raw-as-base64-in-summary | SchemaAnnotationDefinition SchemaFieldDefinition ::= field IdentifierVal type FieldTypeName '{' SchemaFieldBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } FieldTypeName ::= ("array" '<' (FieldTypeName | IdentifierVal) '>') | ("weightedset" '<' SingleValueFieldTypeName '>') | @@ -63,15 +65,15 @@ private TensorDimension ::= WordWrapper (('{' '}') | ('[' INTEGER_REG ']')) SchemaFieldBody ::= DocumentFieldBodyOptions* // Fields of schemas and documents defined the same way here DocumentSummaryDefinition ::= document-summary IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' DocumentSummaryBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } DocumentSummaryBody ::= DocumentSummaryBodyOptions* // Does not support zero-or-one occurrences private DocumentSummaryBodyOptions ::= SummaryDefinition | omit-summary-features | from-disk ImportFieldDefinition ::= import field IdentifierVal '.' IdentifierVal as IdentifierVal '{''}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } FieldSetDefinition ::= fieldset IdentifierVal '{' FieldSetBody '}' @@ -91,14 +93,14 @@ private OnnxModelBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | ((input | output) (IdentifierVal | STRING_REG) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG)) SchemaAnnotationDefinition ::= AnnotationDefinition - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } private AnnotationDefinition ::= annotation IdentifierVal (inherits IdentifierVal)? '{' AnnotationFieldDefinition* '}' AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName '{' '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } //------------------------- @@ -128,8 +130,8 @@ PrimitiveExpr ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | Ra //-- Rank Profile rules --- //------------------------- RankProfileDefinition ::= (rank-profile | model) IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' RankProfileBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration"] } private RankProfileBody ::= RankProfileBodyOptions* // Does not support zero-or-one occurrences private RankProfileBodyOptions ::= MatchPhaseDefinition | NumThreadsDefinition | FunctionDefinition | TermwiseLimitDefinition | @@ -153,7 +155,7 @@ private TermwiseLimitDefinition ::= termwise-limit ':' ('-')? (FLOAT_REG | INTEG private MinHitsDefinition ::= min-hits-per-thread ':' ('-')? INTEGER_REG private NumSearchPartitionDefinition ::= num-search-partition ':' INTEGER_REG FieldWeightDefinition ::= weight IdentifierVal ':' INTEGER_REG -FirstPhaseDefinition ::= first-phase '{' FirstPhaseBody '}' { mixin="org.intellij.sdk.language.psi.impl.SdFirstPhaseDefinitionMixin" } +FirstPhaseDefinition ::= first-phase '{' FirstPhaseBody '}' { mixin="ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin" } FirstPhaseBody ::= FirstPhaseBodyOptions* // Does not support zero-or-one occurrences private FirstPhaseBodyOptions ::= (keep-rank-count ':' INTEGER_REG) | (rank-score-drop-limit ':' ('-')? (FLOAT_REG | INTEGER_REG)) | ExpressionDefinition @@ -168,19 +170,19 @@ RankPropertiesBody ::= (RankPropertiesKey ':' RankPropertiesValue)+ RankPropertiesKey ::= (IdentifierWithDashVal | STRING_REG | '(' | ')' | '.' | ',' | '$' | INTEGER_REG)+ RankPropertiesValue ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | WORD_REG | IdentifierVal | STRING_REG -FunctionDefinition ::= (function | macro) inline? IdentifierVal '(' (ArgumentDefinition (',' ArgumentDefinition)*)? ')' +FunctionDefinition ::= (function | macro) inline? IdentifierVal ( '()' | '(' (ArgumentDefinition (',' ArgumentDefinition)*)? ')' ) '{' ExpressionDefinition '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdFunctionDefinitionInterface" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdFunctionDefinitionInterface" "ai.vespa.intellij.schema.psi.SdNamedElement"] } ArgumentDefinition ::= IdentifierVal - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } SummaryFeaturesDefinition ::= summary-features ((':' RankFeature+) | ((inherits IdentifierVal)? '{' RankFeature* '}')) -MatchFeaturesDefinition ::= match-features ((':' RankFeature+) | ('{' RankFeature* '}')) +MatchFeaturesDefinition ::= match-features ((':' RankFeature+) | ((inherits IdentifierVal)? '{' RankFeature* '}')) RankFeaturesDefinition ::= rank-features ((':' RankFeature+) | ('{' RankFeature* '}')) @@ -188,46 +190,46 @@ ConstantsDefinition ::= constants '{' (IdentifierVal ':' RankPropertiesValue)* ' RankFeature ::= QueryDefinition | ItemRawScoreDefinition | FunctionCallExpr | (IdentifierWithDashVal ('.' IdentifierWithDashVal)* ) QueryDefinition ::= "query" '(' IdentifierWithDashVal ')' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } ItemRawScoreDefinition ::= "itemRawScore" '(' IdentifierVal ')' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } //------------------------- //---- Document rules ----- //------------------------- DocumentDefinition ::= document (IdentifierVal (inherits IdentifierVal (',' IdentifierVal)*)?)? '{' DocumentBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } DocumentBody ::= DocumentBodyOptions* DocumentBodyOptions ::= DocumentStructDefinition | DocumentFieldDefinition | DocumentAnnotationDefinition DocumentAnnotationDefinition ::= AnnotationDefinition - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } DocumentStructDefinition ::= struct IdentifierVal '{' DocumentStructBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } DocumentStructBody ::= DocumentStructFieldDefinition* DocumentStructFieldDefinition ::= field IdentifierVal type FieldTypeName '{' DocumentStructFieldBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } DocumentStructFieldBody ::= MatchDefinition? DocumentFieldDefinition ::= field IdentifierVal type FieldTypeName '{' DocumentFieldBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } DocumentFieldBody ::= DocumentFieldBodyOptions* // Does not support zero-or-one occurrences @@ -235,12 +237,12 @@ private DocumentFieldBodyOptions ::= StructFieldDefinition | MatchDefinition | I AliasDefinition | RankDefinition | IndexingRewriteState | QueryCommandDefinition | SummaryDefinition | BoldingDefinition | (id ':' INTEGER_REG) | IndexDefinition | (normalizing ':' IdentifierWithDashVal) | SortingDefinition | StemmingDefinition | (weight ':' INTEGER_REG) | WeightedSetDefinition | - RankTypeDefinition | DictionaryDefinition | SummaryToDefinition | body + RankTypeDefinition | DictionaryDefinition | SummaryToDefinition | header | body //***** Field's body elements ******// // Struct StructFieldDefinition ::= struct-field IdentifierVal ('.' IdentifierVal)? '{' StructFieldBody '}' - { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl" - implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"] + { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" + implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"] } StructFieldBody ::= StructFieldBodyOptions* // Does not support zero-or-one occurrences @@ -248,15 +250,16 @@ StructFieldBodyOptions ::= IndexingDefinition | AttributeDefinition | MatchDefin StructFieldDefinition | SummaryDefinition // Match MatchDefinition ::= match ((':' MatchProperty) | ('{' MatchProperty+ '}')) -MatchProperty ::= text | exact | (exact-terminator ':' STRING_REG) | word | prefix | cased | uncased | substring | +MatchProperty ::= text | token | exact | (exact-terminator ':' STRING_REG) | word | prefix | cased | uncased | substring | suffix | (max-length ':' INTEGER_REG) | gram | (gram-size ':' INTEGER_REG) | WordWrapper // Indexing IndexingDefinition ::= indexing ((':' IndexingStatement) | ('{' IndexingStatement+ '}')) -IndexingStatement ::= IndexingStatementOptions ((('|' | ';') IndexingStatementOptions)*) +IndexingStatement ::= IndexingStatementOptions (('|' IndexingStatementOptions)*) (';')? // Does not support zero-or-one occurrences -IndexingStatementOptions ::= summary | attribute | index | set_language | lowercase | (input (IdentifierVal | IndexingStuff)+) | +IndexingStatementOptions ::= summary | attribute | index | set_language | lowercase | + (input (IdentifierVal | IndexingStuff)+) | ('{' IndexingStatementOptions '}') | IndexingStuff+ -private IndexingStuff ::= WordWrapper | INTEGER_REG | FLOAT_REG | STRING_REG | ('{' IndexingStatementOptions+ '}') | +private IndexingStuff ::= WordWrapper | INTEGER_REG | FLOAT_REG | STRING_REG | ('{' IndexingStatement '}') | ':' | ('|' IndexingStatementOptions) | ';' | '.' | '(' | ')' | ARITHMETIC_OPERATOR | COMPARISON_OPERATOR // Attribute AttributeDefinition ::= attribute ((':' SimpleAttributeProperty) | ('{' (ComplexAttributeProperty | SimpleAttributeProperty)+ '}')) @@ -264,7 +267,7 @@ SimpleAttributeProperty ::= fast-search | fast-access | paged | mutable | enable ComplexAttributeProperty ::= AliasDefinition | SortingDefinition | DistanceMetricDef // Does not support zero-or-one occurrences DistanceMetricDef ::= distance-metric ':' IdentifierWithDashVal // Alias -AliasDefinition ::= alias (IdentifierWithDashVal ('.' IdentifierWithDashVal)*)? ':' IdentifierWithDashVal ('.' IdentifierWithDashVal)* +AliasDefinition ::= alias (IdentifierVal)? ':' IdentifierWithDashVal ('.' IdentifierWithDashVal)* // Stemming StemmingDefinition ::= stemming ':' IdentifierWithDashVal // Rank @@ -276,7 +279,7 @@ IndexingRewriteState ::= indexing-rewrite ':' none QueryCommandDefinition ::= query-command ':' (IdentifierVal | STRING_REG | WordWrapper) // Summary SummaryDefinition ::= summary IdentifierWithDashVal? (type FieldTypeName)? ((':' SummaryBodyOptions) | ( '{' SummaryBody '}')) - { mixin="org.intellij.sdk.language.psi.impl.SdSummaryDefinitionMixin" } + { mixin="ai.vespa.intellij.schema.psi.impl.SdSummaryDefinitionMixin" } SummaryBody ::= SummaryBodyOptions* // Does not support zero-or-one occurrences SummaryBodyOptions ::= full | static | dynamic | (source ':' (IdentifierVal ('.' IdentifierVal)?) (',' IdentifierVal ('.' IdentifierVal)?)*) | (to ':' IdentifierVal (',' IdentifierVal)*) | matched-elements-only | BoldingDefinition @@ -317,12 +320,12 @@ DictionarySetting ::= hash | btree | cased | uncased private WordWrapper ::= KeywordOrIdentifier | KeywordNotIdentifier | ID_REG | ID_WITH_DASH_REG | WORD_REG -IdentifierVal ::= KeywordOrIdentifier | ID_REG { mixin="org.intellij.sdk.language.psi.impl.SdIdentifierMixin" - implements=["org.intellij.sdk.language.psi.SdIdentifier"] +IdentifierVal ::= KeywordOrIdentifier | ID_REG { mixin="ai.vespa.intellij.schema.psi.impl.SdIdentifierMixin" + implements=["ai.vespa.intellij.schema.psi.SdIdentifier"] } -IdentifierWithDashVal ::= ID_WITH_DASH_REG | IdentifierVal { mixin="org.intellij.sdk.language.psi.impl.SdIdentifierMixin" - implements=["org.intellij.sdk.language.psi.SdIdentifier"] +IdentifierWithDashVal ::= ID_WITH_DASH_REG | IdentifierVal { mixin="ai.vespa.intellij.schema.psi.impl.SdIdentifierMixin" + implements=["ai.vespa.intellij.schema.psi.SdIdentifier"] } // Those lists of keywords (KeywordOrIdentifier and KeywordNotIdentifier) have to be synchronized with sd.flex file. diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java index 99bebe21b87..01992ef4a5b 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.navigation.ChooseByNameContributor; import com.intellij.navigation.NavigationItem; @@ -8,8 +8,8 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiManager; import com.intellij.psi.search.FileTypeIndex; import com.intellij.psi.search.GlobalSearchScope; -import org.intellij.sdk.language.psi.SdDeclaration; -import org.intellij.sdk.language.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdDeclaration; +import ai.vespa.intellij.schema.psi.SdFile; import org.jetbrains.annotations.NotNull; import java.util.ArrayList; @@ -18,6 +18,7 @@ import java.util.List; /** * This class is used for the extension (in plugin.xml) to enable "Go To Symbol" feature. + * * @author Shahar Ariel */ public class SdChooseByNameContributor implements ChooseByNameContributor { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettings.java index 93c3e7ccc50..bdf1bf3eb7b 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettings.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.psi.codeStyle.CodeStyleSettings; import com.intellij.psi.codeStyle.CustomCodeStyleSettings; /** * This class represent a code style settings, and creates an option page in settings/preferences. + * * @author Shahar Ariel */ public class SdCodeStyleSettings extends CustomCodeStyleSettings { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java index 3b075b0f438..b02e8371b77 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.application.options.CodeStyleAbstractConfigurable; import com.intellij.application.options.CodeStyleAbstractPanel; @@ -13,6 +13,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdCodeStyleSettings. + * * @author Shahar Ariel */ public class SdCodeStyleSettingsProvider extends CodeStyleSettingsProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java index d0f28848cf0..a8872f71b63 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.lang.Commenter; import org.jetbrains.annotations.Nullable; @@ -7,6 +7,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable turning a line into a comment with * "Code -> Comment with line comment". + * * @author Shahar Ariel */ public class SdCommenter implements Commenter { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java index 8bcda614ee7..3e5b53c87ea 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.codeInsight.completion.CompletionContributor; import com.intellij.codeInsight.completion.CompletionParameters; @@ -9,11 +9,12 @@ import com.intellij.codeInsight.completion.CompletionType; import com.intellij.codeInsight.lookup.LookupElementBuilder; import com.intellij.patterns.PlatformPatterns; import com.intellij.util.ProcessingContext; -import org.intellij.sdk.language.psi.SdTypes; +import ai.vespa.intellij.schema.psi.SdTypes; import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml) to enables Auto-Complete. Partially works for now. + * * @author Shahar Ariel */ public class SdCompletionContributor extends CompletionContributor { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java index ccc4c6b6348..b469d0c0ebb 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.openapi.fileTypes.LanguageFileType; import org.jetbrains.annotations.NotNull; @@ -9,6 +9,7 @@ import javax.swing.Icon; /** * This class is used for the extension (in plugin.xml), to define SD as a file's type. + * * @author Shahar Ariel */ public class SdFileType extends LanguageFileType { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdIcons.java index 806fcd6b6ba..96f8428b223 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdIcons.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.openapi.util.IconLoader; @@ -7,9 +7,11 @@ import javax.swing.Icon; /** * This class is used for defining Icons for the IDE. + * * @author Shahar Ariel */ public class SdIcons { + public static final Icon FILE = IconLoader.getIcon("icons/sd_icon.png", SdIcons.class); public static final Icon STRUCT_FIELD = IconLoader.getIcon("icons/struct_field_icon.png", SdIcons.class); public static final Icon IMPORTED_FIELD = IconLoader.getIcon("icons/imported_field_icon.png", SdIcons.class); @@ -20,5 +22,6 @@ public class SdIcons { public static final Icon MACRO = IconLoader.getIcon("icons/macro_icon.png", SdIcons.class); public static final Icon OVERRIDE_MACRO = IconLoader.getIcon("icons/override_macro_icon.png", SdIcons.class); public static final Icon FIRST_PHASE = IconLoader.getIcon("icons/first_phase_icon.png", SdIcons.class); + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguage.java index fa253c4d605..c4d471ee737 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguage.java @@ -1,13 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.lang.Language; /** * This class defines SD as a language. + * * @author Shahar Ariel */ public class SdLanguage extends Language { + public static final SdLanguage INSTANCE = new SdLanguage(); private SdLanguage() { @@ -18,4 +20,5 @@ public class SdLanguage extends Language { public boolean isCaseSensitive() { return true; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java index 24fef3af40c..5f618a62d6e 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.lang.Language; import com.intellij.psi.codeStyle.CodeStyleSettingsCustomizable; @@ -8,6 +8,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml), to make the IDE use our plugin's code for coding style. + * * @author Shahar Ariel */ public class SdLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettingsProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java index 1b5cba028a7..24de7bb6253 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java @@ -1,19 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.lang.refactoring.RefactoringSupportProvider; import com.intellij.psi.PsiElement; -import org.intellij.sdk.language.psi.SdIdentifier; +import ai.vespa.intellij.schema.psi.SdIdentifier; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable refactoring. + * * @author Shahar Ariel */ public class SdRefactoringSupportProvider extends RefactoringSupportProvider { + @Override public boolean isMemberInplaceRenameAvailable(@NotNull PsiElement elementToRename, @Nullable PsiElement context) { return (elementToRename instanceof SdIdentifier); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java index bce51ccc2ff..6404a738e59 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.codeInsight.lookup.LookupElement; import com.intellij.codeInsight.lookup.LookupElementBuilder; @@ -11,7 +11,7 @@ import com.intellij.psi.PsiPolyVariantReference; import com.intellij.psi.PsiReferenceBase; import com.intellij.psi.ResolveResult; import com.intellij.util.IncorrectOperationException; -import org.intellij.sdk.language.psi.SdDeclaration; +import ai.vespa.intellij.schema.psi.SdDeclaration; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -19,7 +19,8 @@ import java.util.ArrayList; import java.util.List; /** - * This class represent a reference to a Psi Element. + * A reference to a Psi Element. + * * @author Shahar Ariel */ public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPolyVariantReference { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java index e6452685e3c..df3da81e9fc 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import static com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey; @@ -10,14 +10,15 @@ import com.intellij.openapi.editor.colors.TextAttributesKey; import com.intellij.openapi.fileTypes.SyntaxHighlighterBase; import com.intellij.psi.TokenType; import com.intellij.psi.tree.IElementType; -import org.intellij.sdk.language.lexer.SdLexerAdapter; -import org.intellij.sdk.language.psi.SdTypes; +import ai.vespa.intellij.schema.lexer.SdLexerAdapter; +import ai.vespa.intellij.schema.psi.SdTypes; import org.jetbrains.annotations.NotNull; import java.util.HashSet; /** - * This class defines the syntax highlighting of an SD file. + * Defines the syntax highlighting of an SD file. + * * @author Shahar Ariel */ public class SdSyntaxHighlighter extends SyntaxHighlighterBase { @@ -204,8 +205,8 @@ public class SdSyntaxHighlighter extends SyntaxHighlighterBase { constants.add(SdTypes.LOCALE); constants.add(SdTypes.CREATE_IF_NONEXISTENT); constants.add(SdTypes.REMOVE_IF_ZERO); - return constants; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java index 618e188f680..6cf30ecb54e 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.openapi.fileTypes.SyntaxHighlighter; import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory; @@ -9,6 +9,7 @@ import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml) to the class SdSyntaxHighlighter. + * * @author Shahar Ariel */ public class SdSyntaxHighlighterFactory extends SyntaxHighlighterFactory { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java index 5d6cd85489f..f3f949d645f 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language; +package ai.vespa.intellij.schema; import com.intellij.lang.ASTNode; import com.intellij.openapi.project.Project; @@ -11,25 +11,25 @@ import com.intellij.psi.PsiReference; import com.intellij.psi.search.FileTypeIndex; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition; -import org.intellij.sdk.language.psi.SdArgumentDefinition; -import org.intellij.sdk.language.psi.SdDeclaration; -import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition; -import org.intellij.sdk.language.psi.SdDocumentDefinition; -import org.intellij.sdk.language.psi.SdDocumentFieldDefinition; -import org.intellij.sdk.language.psi.SdDocumentStructDefinition; -import org.intellij.sdk.language.psi.SdDocumentStructFieldDefinition; -import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; -import org.intellij.sdk.language.psi.SdFieldTypeName; -import org.intellij.sdk.language.psi.SdFile; -import org.intellij.sdk.language.psi.SdFunctionDefinition; -import org.intellij.sdk.language.psi.SdIdentifier; -import org.intellij.sdk.language.psi.SdImportFieldDefinition; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; -import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition; -import org.intellij.sdk.language.psi.SdSchemaFieldDefinition; -import org.intellij.sdk.language.psi.SdSummaryDefinition; -import org.intellij.sdk.language.psi.SdTypes; +import ai.vespa.intellij.schema.psi.SdAnnotationFieldDefinition; +import ai.vespa.intellij.schema.psi.SdArgumentDefinition; +import ai.vespa.intellij.schema.psi.SdDeclaration; +import ai.vespa.intellij.schema.psi.SdDocumentAnnotationDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentFieldDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentStructDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentStructFieldDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition; +import ai.vespa.intellij.schema.psi.SdFieldTypeName; +import ai.vespa.intellij.schema.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdIdentifier; +import ai.vespa.intellij.schema.psi.SdImportFieldDefinition; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition; +import ai.vespa.intellij.schema.psi.SdSchemaFieldDefinition; +import ai.vespa.intellij.schema.psi.SdSummaryDefinition; +import ai.vespa.intellij.schema.psi.SdTypes; import org.jetbrains.annotations.NotNull; import java.util.ArrayList; @@ -38,7 +38,8 @@ import java.util.HashMap; import java.util.List; /** - * This is the util class for the plugin's code. + * Util class for the plugin's code. + * * @author Shahar Ariel */ public class SdUtil { @@ -217,5 +218,4 @@ public class SdUtil { return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdSummaryDefinition.class)); } - } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java index ba081e9d415..ab463562355 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.openapi.project.DumbAware; import com.intellij.psi.PsiElement; @@ -8,13 +8,14 @@ import com.intellij.usages.UsageGroup; import com.intellij.usages.UsageTarget; import com.intellij.usages.rules.PsiElementUsage; import com.intellij.usages.rules.SingleParentUsageGroupingRule; -import org.intellij.sdk.language.SdLanguage; -import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; +import ai.vespa.intellij.schema.SdLanguage; +import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** - * This class represent a Document Summary that groups elements in the "Find Usages" window. + * A Document Summary that groups elements in the "Find Usages" window. + * * @author Shahar Ariel */ public class SdDocumentSummaryGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java index 0e6f0358ca5..76b82d7c857 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.openapi.project.Project; import com.intellij.usages.impl.FileStructureGroupRuleProvider; @@ -9,6 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdDocumentSummaryGroupingRule. + * * @author Shahar Ariel */ public class SdDocumentSummaryGroupingRuleProvider implements FileStructureGroupRuleProvider { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java index b48c35e5c49..4609f2e7fbb 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.find.findUsages.FindUsagesHandler; import com.intellij.find.findUsages.FindUsagesOptions; @@ -13,9 +13,9 @@ import com.intellij.psi.search.SearchScope; import com.intellij.psi.search.searches.ReferencesSearch; import com.intellij.usageView.UsageInfo; import com.intellij.util.Processor; -import org.intellij.sdk.language.SdUtil; -import org.intellij.sdk.language.psi.SdFile; -import org.intellij.sdk.language.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.SdUtil; +import ai.vespa.intellij.schema.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import org.jetbrains.annotations.NotNull; import java.util.HashMap; @@ -23,6 +23,7 @@ import java.util.List; /** * This class handles creating the "Find Usages" window. + * * @author Shahar Ariel */ public class SdFindUsagesHandler extends FindUsagesHandler { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java index b96cad508f7..98be15778e7 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.find.findUsages.FindUsagesHandler; import com.intellij.find.findUsages.FindUsagesHandlerFactory; @@ -10,6 +10,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdFindUsagesHandler. + * * @author Shahar Ariel */ public class SdFindUsagesHandlerFactory extends FindUsagesHandlerFactory { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java index e3eb6b431c7..a86fb2b058b 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.lang.cacheBuilder.DefaultWordsScanner; import com.intellij.lang.cacheBuilder.WordsScanner; @@ -7,20 +7,22 @@ import com.intellij.lang.findUsages.FindUsagesProvider; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; import com.intellij.psi.tree.TokenSet; -import org.intellij.sdk.language.lexer.SdLexerAdapter; -import org.intellij.sdk.language.psi.SdDeclaration; -import org.intellij.sdk.language.psi.SdIdentifierVal; -import org.intellij.sdk.language.psi.SdIdentifierWithDashVal; -import org.intellij.sdk.language.psi.SdTypes; -import org.intellij.sdk.language.psi.impl.SdNamedElementImpl; +import ai.vespa.intellij.schema.lexer.SdLexerAdapter; +import ai.vespa.intellij.schema.psi.SdDeclaration; +import ai.vespa.intellij.schema.psi.SdIdentifierVal; +import ai.vespa.intellij.schema.psi.SdIdentifierWithDashVal; +import ai.vespa.intellij.schema.psi.SdTypes; +import ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable "find Usages" window using the plugin code. + * * @author Shahar Ariel */ public class SdFindUsagesProvider implements FindUsagesProvider { + @Nullable @Override public WordsScanner getWordsScanner() { @@ -71,4 +73,5 @@ public class SdFindUsagesProvider implements FindUsagesProvider { return ""; } } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java index 1fa848313e0..fd3073b6016 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.openapi.project.DumbAware; import com.intellij.psi.PsiElement; @@ -8,13 +8,14 @@ import com.intellij.usages.UsageGroup; import com.intellij.usages.UsageTarget; import com.intellij.usages.rules.PsiElementUsage; import com.intellij.usages.rules.SingleParentUsageGroupingRule; -import org.intellij.sdk.language.SdLanguage; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.SdLanguage; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** - * This class represent a Rank Profile that groups elements in the "Find Usages" window. + * A Rank Profile that groups elements in the "Find Usages" window. + * * @author Shahar Ariel */ public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule implements DumbAware { @@ -34,4 +35,5 @@ public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule imp return null; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java index 0b973a56644..0726e103943 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.openapi.project.Project; import com.intellij.usages.impl.FileStructureGroupRuleProvider; @@ -9,6 +9,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdRankProfileGroupingRule. + * * @author Shahar Ariel */ public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRuleProvider { @@ -17,4 +18,5 @@ public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRule public @Nullable UsageGroupingRule getUsageGroupingRule(@NotNull Project project) { return new SdRankProfileGroupingRule(); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java index b67c561e8d4..6f24ce27476 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.findUsages; +package ai.vespa.intellij.schema.findUsages; import com.intellij.navigation.ItemPresentation; import com.intellij.navigation.NavigationItemFileStatus; @@ -10,17 +10,19 @@ import com.intellij.psi.SmartPointerManager; import com.intellij.psi.SmartPsiElementPointer; import com.intellij.usages.UsageGroup; import com.intellij.usages.UsageView; -import org.intellij.sdk.language.psi.SdDeclaration; +import ai.vespa.intellij.schema.psi.SdDeclaration; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; /** - * This class represent a group of elements in the "Find Usages" window. + * A group of elements in the "Find Usages" window. + * * @author Shahar Ariel */ public class SdUsageGroup implements UsageGroup { + private final VirtualFile myFile; private final SmartPsiElementPointer<SdDeclaration> myElementPointer; private final String myText; @@ -107,4 +109,5 @@ public class SdUsageGroup implements UsageGroup { @Override public void update() {} + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java index 46b4e738fcb..38817e15553 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; +package ai.vespa.intellij.schema.hierarchy; import com.intellij.ide.hierarchy.CallHierarchyBrowserBase; import com.intellij.ide.hierarchy.HierarchyNodeDescriptor; @@ -11,7 +11,7 @@ import com.intellij.openapi.actionSystem.IdeActions; import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; import com.intellij.util.ObjectUtils; -import org.intellij.sdk.language.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -21,7 +21,8 @@ import java.util.Map; import javax.swing.JTree; /** - * This class is a browser for the "Call Hierarchy" window. + * A browser for the "Call Hierarchy" window. + * * @author Shahar Ariel */ public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java index fdd8871201e..f99a82dc81a 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; +package ai.vespa.intellij.schema.hierarchy; import com.intellij.ide.hierarchy.HierarchyNodeDescriptor; import com.intellij.ide.util.treeView.NodeDescriptor; @@ -10,18 +10,19 @@ import com.intellij.psi.NavigatablePsiElement; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; -import org.intellij.sdk.language.SdIcons; -import org.intellij.sdk.language.SdUtil; -import org.intellij.sdk.language.psi.SdFirstPhaseDefinition; -import org.intellij.sdk.language.psi.impl.SdFirstPhaseDefinitionMixin; -import org.intellij.sdk.language.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.SdIcons; +import ai.vespa.intellij.schema.SdUtil; +import ai.vespa.intellij.schema.psi.SdFirstPhaseDefinition; +import ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import org.jetbrains.annotations.NotNull; import javax.swing.Icon; /** - * This class represents a node descriptor to a node in a tree in the "Call Hierarchy" window. + * A node descriptor to a node in a tree in the "Call Hierarchy" window. + * * @author Shahar Ariel */ public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor { @@ -68,10 +69,7 @@ public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor { if (!Comparing.equal(myHighlightedText, oldText)) { changes = true; } - - return changes; - } } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java index 4808c55f1ae..f7f243356b9 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; - +package ai.vespa.intellij.schema.hierarchy; import com.intellij.ide.hierarchy.CallHierarchyBrowserBase; import com.intellij.ide.hierarchy.HierarchyBrowser; @@ -11,8 +10,8 @@ import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiReference; -import org.intellij.sdk.language.psi.SdFunctionDefinition; -import org.intellij.sdk.language.psi.SdIdentifierVal; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdIdentifierVal; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import com.intellij.openapi.actionSystem.CommonDataKeys; @@ -20,6 +19,7 @@ import com.intellij.openapi.project.Project; /** * This class is used for the extension (in plugin.xml), to enable "Call Hierarchy" window using the plugin code. + * * @author Shahar Ariel */ public class SdCallHierarchyProvider implements HierarchyProvider { @@ -61,4 +61,5 @@ public class SdCallHierarchyProvider implements HierarchyProvider { public void browserActivated(@NotNull HierarchyBrowser hierarchyBrowser) { ((SdCallHierarchyBrowser) hierarchyBrowser).changeView(CallHierarchyBrowserBase.getCallerType()); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java index 0f843d9fe6a..053e7bd20f9 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; +package ai.vespa.intellij.schema.hierarchy; import com.intellij.ide.hierarchy.HierarchyNodeDescriptor; import com.intellij.ide.hierarchy.HierarchyTreeStructure; @@ -8,10 +8,10 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.ArrayUtilRt; -import org.intellij.sdk.language.SdUtil; -import org.intellij.sdk.language.psi.SdFile; -import org.intellij.sdk.language.psi.SdFunctionDefinition; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.SdUtil; +import ai.vespa.intellij.schema.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; import org.jetbrains.annotations.NotNull; import java.util.ArrayList; @@ -20,10 +20,12 @@ import java.util.HashSet; import java.util.List; /** - * This abstract class represents a general tree in the "Call Hierarchy" window. + * A general tree in the "Call Hierarchy" window. + * * @author Shahar Ariel */ public abstract class SdCallTreeStructure extends HierarchyTreeStructure { + protected final String myScopeType; protected final SdFile myFile; protected HashMap<String, List<PsiElement>> macrosMap; @@ -73,4 +75,5 @@ public abstract class SdCallTreeStructure extends HierarchyTreeStructure { } return ArrayUtil.toObjectArray(descriptors); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java index cce1ebf172a..37fd336b615 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; +package ai.vespa.intellij.schema.hierarchy; import com.intellij.openapi.project.Project; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; import com.intellij.psi.PsiReference; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.psi.SdExpressionDefinition; -import org.intellij.sdk.language.psi.SdFunctionDefinition; -import org.intellij.sdk.language.psi.SdIdentifier; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.psi.SdExpressionDefinition; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdIdentifier; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; import org.jetbrains.annotations.NotNull; import java.util.HashMap; @@ -17,7 +17,8 @@ import java.util.HashSet; import java.util.List; /** - * This class represents a Callee tree in the "Call Hierarchy" window. + * A Callee tree in the "Call Hierarchy" window. + * * @author Shahar Ariel */ public class SdCalleeTreeStructure extends SdCallTreeStructure { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java index 1ab2f6613d9..444e71978c9 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; +package ai.vespa.intellij.schema.hierarchy; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.Project; @@ -9,8 +9,8 @@ import com.intellij.psi.search.SearchScope; import com.intellij.psi.search.searches.ReferencesSearch; import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.containers.ContainerUtil; -import org.intellij.sdk.language.psi.SdFirstPhaseDefinition; -import org.intellij.sdk.language.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdFirstPhaseDefinition; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; import org.jetbrains.annotations.NotNull; import java.util.HashMap; @@ -19,7 +19,8 @@ import java.util.List; import java.util.function.Consumer; /** - * This class represents a Caller tree in the "Call Hierarchy" window. + * A Caller tree in the "Call Hierarchy" window. + * * @author Shahar Ariel */ public class SdCallerTreeStructure extends SdCallTreeStructure { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java index 19f9d72f6f4..dc925f0f369 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.hierarchy; +package ai.vespa.intellij.schema.hierarchy; import com.intellij.ide.hierarchy.HierarchyBrowserManager; import com.intellij.ide.util.treeView.AlphaComparator; @@ -8,16 +8,17 @@ import com.intellij.openapi.project.Project; import com.intellij.psi.*; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.SdUtil; -import org.intellij.sdk.language.psi.SdFile; -import org.intellij.sdk.language.psi.SdFunctionDefinition; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.SdUtil; +import ai.vespa.intellij.schema.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; import java.util.Comparator; import java.util.HashSet; /** - * This class is util class to the Call Hierarchy feature. + * Call Hierarchy feature utilities. + * * @author Shahar Ariel */ public class SdHierarchyUtil { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/SdLexerAdapter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/lexer/SdLexerAdapter.java index c7463fc9823..8b474dcff69 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/SdLexerAdapter.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/lexer/SdLexerAdapter.java @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.lexer; +package ai.vespa.intellij.schema.lexer; import com.intellij.lexer.FlexAdapter; /** - * This class adapts the JFlex lexer to the IntelliJ Platform Lexer API. + * Adapter of the JFlex lexer to the IntelliJ Platform Lexer API. + * * @author Shahar Ariel */ public class SdLexerAdapter extends FlexAdapter { @@ -12,4 +13,5 @@ public class SdLexerAdapter extends FlexAdapter { public SdLexerAdapter() { super(new SdLexer(null)); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java index 0497f0b4960..d5b76c039d0 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.parser; +package ai.vespa.intellij.schema.parser; import com.intellij.lang.ASTNode; import com.intellij.lang.ParserDefinition; @@ -12,17 +12,19 @@ import com.intellij.psi.PsiFile; import com.intellij.psi.TokenType; import com.intellij.psi.tree.IFileElementType; import com.intellij.psi.tree.TokenSet; -import org.intellij.sdk.language.SdLanguage; -import org.intellij.sdk.language.lexer.SdLexerAdapter; -import org.intellij.sdk.language.psi.SdFile; -import org.intellij.sdk.language.psi.SdTypes; +import ai.vespa.intellij.schema.SdLanguage; +import ai.vespa.intellij.schema.lexer.SdLexerAdapter; +import ai.vespa.intellij.schema.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdTypes; import org.jetbrains.annotations.NotNull; /** * This class is used for the extension (in plugin.xml), to make the parsing process use the plugin code. + * * @author Shahar Ariel */ public class SdParserDefinition implements ParserDefinition { + public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE); public static final TokenSet COMMENTS = TokenSet.create(SdTypes.COMMENT); public static final TokenSet STRINGS = TokenSet.create(SdTypes.STRING_REG); diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclaration.java index 4f11a2b4b67..c2880181d66 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclaration.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.navigation.NavigationItem; import com.intellij.psi.PsiElement; /** - * This interface represents a declaration in the SD language. + * A declaration in the SD language. + * * @author Shahar Ariel */ public interface SdDeclaration extends PsiElement, NavigationItem, SdNamedElement {} diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclarationType.java index c19e21d0f19..129a32153f0 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclarationType.java @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; /** * This Enum describes the different declarations' types and their names. + * * @author Shahar Ariel */ public enum SdDeclarationType { + DOCUMENT("Document"), STRUCT("Struct"), ANNOTATION("Annotation"), @@ -31,4 +33,5 @@ public enum SdDeclarationType { public String toString() { return typeName; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java index 76c4454b3b2..7cd2d7f9331 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java @@ -1,22 +1,24 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.psi.ElementDescriptionLocation; import com.intellij.psi.ElementDescriptionProvider; import com.intellij.psi.PsiElement; -import org.intellij.sdk.language.psi.impl.SdNamedElementImpl; +import ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml), to enable "find Usages" window take the element description from * here. Used only for the "target" element. + * * @author Shahar Ariel */ public class SdElementDescriptionProvider implements ElementDescriptionProvider { /** - * Controls the headline of the element in the "Find Usages" window + * Controls the headline of the element in the "Find Usages" window. + * * @param psiElement the element to describe * @return a string with the description to write in the headline */ @@ -29,4 +31,5 @@ public class SdElementDescriptionProvider implements ElementDescriptionProvider return ""; } } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementFactory.java index 1f3b98b6012..d6cbcece6d5 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementFactory.java @@ -1,14 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.openapi.project.Project; import com.intellij.psi.PsiFileFactory; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.SdFileType; +import ai.vespa.intellij.schema.SdFileType; /** * This class is a factory of psi elements in the SD PSI tree. + * * @author Shahar Ariel */ public class SdElementFactory { @@ -32,4 +33,5 @@ public class SdElementFactory { return (SdFile) PsiFileFactory.getInstance(project). createFileFromText(name, SdFileType.INSTANCE, text); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java index b26cd66455d..e8154bce8ad 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java @@ -1,13 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.psi.tree.IElementType; -import org.intellij.sdk.language.SdLanguage; +import ai.vespa.intellij.schema.SdLanguage; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; /** - * This class represent a SdElementType. + * An SdElementType. + * * @author Shahar Ariel */ public class SdElementType extends IElementType { @@ -15,4 +16,5 @@ public class SdElementType extends IElementType { public SdElementType(@NotNull @NonNls String debugName) { super(debugName, SdLanguage.INSTANCE); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java index c2d9249bce1..1260883c96f 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java @@ -1,15 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.extapi.psi.PsiFileBase; import com.intellij.openapi.fileTypes.FileType; import com.intellij.psi.FileViewProvider; -import org.intellij.sdk.language.SdFileType; -import org.intellij.sdk.language.SdLanguage; +import ai.vespa.intellij.schema.SdFileType; +import ai.vespa.intellij.schema.SdLanguage; import org.jetbrains.annotations.NotNull; /** - * This class represent an SD file. + * An SD file. + * * @author Shahar Ariel */ public class SdFile extends PsiFileBase { @@ -28,4 +29,5 @@ public class SdFile extends PsiFileBase { public String toString() { return "Sd File"; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java index 78d964b45c9..e7fa94cf81c 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java @@ -1,13 +1,15 @@ -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.SdUtil; +import ai.vespa.intellij.schema.SdUtil; /** - * This interface represents a function's declaration in the SD language. + * A function's declaration in the SD language. + * * @author Shahar Ariel */ public interface SdFunctionDefinitionInterface extends SdDeclaration { + default boolean isOverride() { String macroName = this.getName(); @@ -24,6 +26,6 @@ public interface SdFunctionDefinitionInterface extends SdDeclaration { curRankProfile = (SdRankProfileDefinition) SdUtil.getRankProfileParent(curRankProfile); } return false; - } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdIdentifier.java index e20eb4cb6c4..1ac88cb3acd 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdIdentifier.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.psi.PsiElement; import com.intellij.psi.util.PsiTreeUtil; diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdNamedElement.java index 6e56f548410..0cd7cedc967 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdNamedElement.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.psi.PsiNameIdentifierOwner; /** * This interface is used to wrap a Psi Element with SdNamedElement interface, which enables the element to be a * "name owner" (like an identifier). It allows the element to take a part in references, find usages and more. + * * @author Shahar Ariel */ public interface SdNamedElement extends PsiNameIdentifierOwner { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java index 1024995832d..e42906530b0 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java @@ -1,13 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi; +package ai.vespa.intellij.schema.psi; import com.intellij.psi.tree.IElementType; -import org.intellij.sdk.language.SdLanguage; +import ai.vespa.intellij.schema.SdLanguage; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; /** - * This class represent a SdTokenType. + * An SdTokenType. + * * @author Shahar Ariel */ public class SdTokenType extends IElementType { @@ -20,4 +21,5 @@ public class SdTokenType extends IElementType { public String toString() { return "SdTokenType." + super.toString(); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java index 1a13318a38f..43c02ea4fcf 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java @@ -1,11 +1,12 @@ -package org.intellij.sdk.language.psi.impl; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.intellij.schema.psi.impl; import com.intellij.extapi.psi.ASTWrapperPsiElement; import com.intellij.lang.ASTNode; import com.intellij.navigation.ItemPresentation; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.SdIcons; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.SdIcons; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -14,6 +15,7 @@ import javax.swing.Icon; /** * This class is used for methods' implementations for SdFirstPhaseDefinition. Connected with "mixin" to * FirstPhaseDefinition rule in sd.bnf + * * @author Shahar Ariel */ public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement { @@ -56,3 +58,4 @@ public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement { }; } } + diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java index 536dfc4f2cc..897217c082c 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java @@ -1,16 +1,18 @@ -package org.intellij.sdk.language.psi.impl; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.intellij.schema.psi.impl; import com.intellij.lang.ASTNode; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiNamedElement; -import org.intellij.sdk.language.psi.SdElementFactory; -import org.intellij.sdk.language.psi.SdIdentifier; -import org.intellij.sdk.language.psi.SdIdentifierVal; +import ai.vespa.intellij.schema.psi.SdElementFactory; +import ai.vespa.intellij.schema.psi.SdIdentifier; +import ai.vespa.intellij.schema.psi.SdIdentifierVal; import org.jetbrains.annotations.NotNull; /** * This abstract class is used for methods' implementations for SdIdentifier. Connected with "mixin" to IdentifierVal and * IdentifierWithDashVal rules in sd.bnf + * * @author Shahar Ariel */ public abstract class SdIdentifierMixin extends SdIdentifierMixinImpl implements PsiNamedElement { @@ -41,6 +43,4 @@ public abstract class SdIdentifierMixin extends SdIdentifierMixinImpl implements return this; } - - } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java index 7b738cc1b0f..3f69952d0b6 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java @@ -1,15 +1,17 @@ -package org.intellij.sdk.language.psi.impl; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.intellij.schema.psi.impl; import com.intellij.extapi.psi.ASTWrapperPsiElement; import com.intellij.lang.ASTNode; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiReference; -import org.intellij.sdk.language.SdReference; -import org.intellij.sdk.language.psi.SdIdentifier; +import ai.vespa.intellij.schema.SdReference; +import ai.vespa.intellij.schema.psi.SdIdentifier; import org.jetbrains.annotations.NotNull; /** * This class is used for methods' implementations for SdIdentifier. The abstract class SdIdentifierMixin extents it. + * * @author Shahar Ariel */ public class SdIdentifierMixinImpl extends ASTWrapperPsiElement implements SdIdentifier { @@ -22,4 +24,5 @@ public class SdIdentifierMixinImpl extends ASTWrapperPsiElement implements SdIde public PsiReference getReference() { return new SdReference(this, new TextRange(0, getNode().getText().length())); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java index 2c2a7aae3b1..ee328f43075 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.psi.impl; +package ai.vespa.intellij.schema.psi.impl; import com.intellij.extapi.psi.ASTWrapperPsiElement; import com.intellij.icons.AllIcons; @@ -7,29 +7,29 @@ import com.intellij.lang.ASTNode; import com.intellij.navigation.ItemPresentation; import com.intellij.psi.PsiElement; import com.intellij.psi.util.PsiTreeUtil; -import org.intellij.sdk.language.SdIcons; -import org.intellij.sdk.language.SdUtil; -import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition; -import org.intellij.sdk.language.psi.SdArgumentDefinition; -import org.intellij.sdk.language.psi.SdDeclarationType; -import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition; -import org.intellij.sdk.language.psi.SdDocumentDefinition; -import org.intellij.sdk.language.psi.SdDocumentFieldDefinition; -import org.intellij.sdk.language.psi.SdDocumentStructDefinition; -import org.intellij.sdk.language.psi.SdDocumentStructFieldDefinition; -import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; -import org.intellij.sdk.language.psi.SdElementFactory; -import org.intellij.sdk.language.psi.SdFunctionDefinition; -import org.intellij.sdk.language.psi.SdIdentifier; -import org.intellij.sdk.language.psi.SdImportFieldDefinition; -import org.intellij.sdk.language.psi.SdItemRawScoreDefinition; -import org.intellij.sdk.language.psi.SdNamedElement; -import org.intellij.sdk.language.psi.SdQueryDefinition; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; -import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition; -import org.intellij.sdk.language.psi.SdSchemaFieldDefinition; -import org.intellij.sdk.language.psi.SdStructFieldDefinition; -import org.intellij.sdk.language.psi.SdTypes; +import ai.vespa.intellij.schema.SdIcons; +import ai.vespa.intellij.schema.SdUtil; +import ai.vespa.intellij.schema.psi.SdAnnotationFieldDefinition; +import ai.vespa.intellij.schema.psi.SdArgumentDefinition; +import ai.vespa.intellij.schema.psi.SdDeclarationType; +import ai.vespa.intellij.schema.psi.SdDocumentAnnotationDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentFieldDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentStructDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentStructFieldDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition; +import ai.vespa.intellij.schema.psi.SdElementFactory; +import ai.vespa.intellij.schema.psi.SdFunctionDefinition; +import ai.vespa.intellij.schema.psi.SdIdentifier; +import ai.vespa.intellij.schema.psi.SdImportFieldDefinition; +import ai.vespa.intellij.schema.psi.SdItemRawScoreDefinition; +import ai.vespa.intellij.schema.psi.SdNamedElement; +import ai.vespa.intellij.schema.psi.SdQueryDefinition; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition; +import ai.vespa.intellij.schema.psi.SdSchemaFieldDefinition; +import ai.vespa.intellij.schema.psi.SdStructFieldDefinition; +import ai.vespa.intellij.schema.psi.SdTypes; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -200,4 +200,5 @@ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements } }; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java index 085b1e04102..71d93641939 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java @@ -1,10 +1,11 @@ -package org.intellij.sdk.language.psi.impl; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.intellij.schema.psi.impl; import com.intellij.extapi.psi.ASTWrapperPsiElement; import com.intellij.lang.ASTNode; import com.intellij.navigation.ItemPresentation; -import org.intellij.sdk.language.SdIcons; -import org.intellij.sdk.language.psi.SdTypes; +import ai.vespa.intellij.schema.SdIcons; +import ai.vespa.intellij.schema.psi.SdTypes; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -13,6 +14,7 @@ import javax.swing.Icon; /** * This class is used for methods' implementations for SdSummaryDefinition. Connected with "mixin" to SummaryDefinition * rule in sd.bnf + * * @author Shahar Ariel */ public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement { @@ -54,4 +56,5 @@ public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement { } }; } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java index 67bda9ced19..76662c5cd31 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.structure; +package ai.vespa.intellij.schema.structure; import com.intellij.ide.projectView.PresentationData; import com.intellij.ide.structureView.StructureViewTreeElement; @@ -8,14 +8,14 @@ import com.intellij.ide.util.treeView.smartTree.TreeElement; import com.intellij.navigation.ItemPresentation; import com.intellij.psi.NavigatablePsiElement; import com.intellij.psi.PsiElement; -import org.intellij.sdk.language.SdUtil; -import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition; -import org.intellij.sdk.language.psi.SdDocumentDefinition; -import org.intellij.sdk.language.psi.SdDocumentStructDefinition; -import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition; -import org.intellij.sdk.language.psi.SdFile; -import org.intellij.sdk.language.psi.SdRankProfileDefinition; -import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition; +import ai.vespa.intellij.schema.SdUtil; +import ai.vespa.intellij.schema.psi.SdDocumentAnnotationDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentStructDefinition; +import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition; +import ai.vespa.intellij.schema.psi.SdFile; +import ai.vespa.intellij.schema.psi.SdRankProfileDefinition; +import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition; import org.jetbrains.annotations.NotNull; import java.util.ArrayList; @@ -23,6 +23,7 @@ import java.util.List; /** * This class is used for the presentation of an element in the "Structure View" window. + * * @author Shahar Ariel */ public class SdStructureViewElement implements StructureViewTreeElement, SortableTreeElement { @@ -91,4 +92,5 @@ public class SdStructureViewElement implements StructureViewTreeElement, Sortabl } return treeElements.toArray(new TreeElement[0]); } + } diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java index 2e902beb2ad..3d7b3a1a275 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.structure; +package ai.vespa.intellij.schema.structure; import com.intellij.ide.structureView.StructureViewBuilder; import com.intellij.ide.structureView.StructureViewModel; @@ -13,6 +13,7 @@ import org.jetbrains.annotations.Nullable; /** * This class is used for the extension (in plugin.xml) to the class SdStructureViewModel. It make the IDE use our * plugin's code when creating the "Structure View" window. + * * @author Shahar Ariel */ public class SdStructureViewFactory implements PsiStructureViewFactory { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java index 00cf18565aa..1f1c271dcf6 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java +++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package org.intellij.sdk.language.structure; +package ai.vespa.intellij.schema.structure; import com.intellij.ide.structureView.StructureViewModel; import com.intellij.ide.structureView.StructureViewModelBase; @@ -10,6 +10,8 @@ import org.jetbrains.annotations.NotNull; /** * This class represents the "Structure View" window. + * + * @author Shahar Ariel */ public class SdStructureViewModel extends StructureViewModelBase implements StructureViewModel.ElementInfoProvider { public SdStructureViewModel(PsiFile psiFile) { diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex b/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex index 585a6672189..78334de1961 100644 --- a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex +++ b/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex @@ -1,19 +1,21 @@ -package org.intellij.sdk.language.lexer; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.intellij.schema.lexer; import com.intellij.lexer.FlexLexer; import com.intellij.psi.tree.IElementType; import com.intellij.ui.components.MultiColumnList; -import org.intellij.sdk.language.psi.SdTokenType; +import ai.vespa.intellij.schema.psi.SdTokenType; -import static org.intellij.sdk.language.psi.SdTypes.*; // That is the class which is specified as `elementTypeHolderClass` in bnf +import static ai.vespa.intellij.schema.psi.SdTypes.*; // That is the class which is specified as `elementTypeHolderClass` in bnf // grammar file. This will contain all other tokens which we will use. import static com.intellij.psi.TokenType.BAD_CHARACTER; // Pre-defined bad character token. import static com.intellij.psi.TokenType.WHITE_SPACE; // Pre-defined whitespace character token. /* -* This file is used for the generation of the lexer of the SD language. -* @author Shahar Ariel -*/ + * Vespa schema parser lexer + * + * @author Shahar Ariel + */ %% @@ -36,7 +38,7 @@ SYMBOL= [!$|:{}(),.\[\]] INTEGER = [0-9]+ FLOAT = {INTEGER}[.][0-9]+[e]? COMPARISON_OPERATOR = [<>]|(==)|(<=)|(>=)|(\~=) -ARITHMETIC_OPERATOR = [\-+*/] +ARITHMETIC_OPERATOR = [\-+*/%] STRING = \"([^\"\\]*(\\.[^\"\\]*)*)\" WORD = \w+ diff --git a/integration/intellij/src/main/resources/META-INF/plugin.xml b/integration/intellij/src/main/resources/META-INF/plugin.xml new file mode 100644 index 00000000000..49db6c59b3e --- /dev/null +++ b/integration/intellij/src/main/resources/META-INF/plugin.xml @@ -0,0 +1,41 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<idea-plugin> + <id>ai.vespa</id> + <name>Vespa</name> + + <!-- Text to display as company information on Preferences/Settings | Plugin page --> + <vendor>vespa.ai</vendor> + + <!-- Product and plugin compatibility requirements --> + <depends>com.intellij.java</depends> + <depends>com.intellij.modules.platform</depends> + + <!-- Text to display as description on Preferences/Settings | Plugin page --> + <description><![CDATA[ + <p>Vespa.ai schema file support</p> + ]]></description> + + <!-- Extension points defined by the plugin --> + <extensions defaultExtensionNs="com.intellij"> + <fileType name="Sd File" implementationClass="ai.vespa.intellij.schema.SdFileType" fieldName="INSTANCE" + language="Sd" extensions="sd"/> + <lang.parserDefinition language="Sd" implementationClass="ai.vespa.intellij.schema.parser.SdParserDefinition"/> + <lang.syntaxHighlighterFactory language="Sd" implementationClass="ai.vespa.intellij.schema.SdSyntaxHighlighterFactory"/> + <completion.contributor language="Sd" implementationClass="ai.vespa.intellij.schema.SdCompletionContributor"/> + + <lang.findUsagesProvider language="Sd" implementationClass="ai.vespa.intellij.schema.findUsages.SdFindUsagesProvider"/> + <findUsagesHandlerFactory implementation="ai.vespa.intellij.schema.findUsages.SdFindUsagesHandlerFactory"/> + <fileStructureGroupRuleProvider implementation="ai.vespa.intellij.schema.findUsages.SdRankProfileGroupingRuleProvider"/> + <fileStructureGroupRuleProvider implementation="ai.vespa.intellij.schema.findUsages.SdDocumentSummaryGroupingRuleProvider"/> + <elementDescriptionProvider implementation="ai.vespa.intellij.schema.psi.SdElementDescriptionProvider"/> + + <lang.psiStructureViewFactory language="Sd" implementationClass="ai.vespa.intellij.schema.structure.SdStructureViewFactory"/> + <codeStyleSettingsProvider implementation="ai.vespa.intellij.schema.SdCodeStyleSettingsProvider"/> + <langCodeStyleSettingsProvider implementation="ai.vespa.intellij.schema.SdLanguageCodeStyleSettingsProvider"/> + <lang.commenter language="Sd" implementationClass="ai.vespa.intellij.schema.SdCommenter"/> + <lang.refactoringSupport language="Sd" implementationClass="ai.vespa.intellij.schema.SdRefactoringSupportProvider"/> + <gotoSymbolContributor implementation="ai.vespa.intellij.schema.SdChooseByNameContributor"/> + <callHierarchyProvider language="Sd" implementationClass="ai.vespa.intellij.schema.hierarchy.SdCallHierarchyProvider"/> + </extensions> + +</idea-plugin>
\ No newline at end of file diff --git a/integration/intellij/src/main/resources/META-INF/pluginIcon.svg b/integration/intellij/src/main/resources/META-INF/pluginIcon.svg new file mode 100644 index 00000000000..3a795976efd --- /dev/null +++ b/integration/intellij/src/main/resources/META-INF/pluginIcon.svg @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="utf-8"?> +<svg id="Layer_1" data-name="Layer 1" viewBox="0 0 40 40" preserveAspectRatio="xMidYMin slice" xmlns="http://www.w3.org/2000/svg"> + <defs> + <style>.cls-1{fill:#ffc43c;}.cls-2{fill:url(#linear-gradient);}.cls-3{fill:#1a7db6;}.cls-4{fill:url(#linear-gradient-2);}.cls-5{fill:#303030;}</style> + <linearGradient id="linear-gradient" x1="964.75" y1="34.82" x2="919.68" y2="63.4" gradientTransform="matrix(-0.276923, 0, 0, 0.276923, 291.653198, 2)" gradientUnits="userSpaceOnUse"> + <stop offset="0" stop-color="#da9728"/> + <stop offset="0.54" stop-color="#ffc43c"/> + </linearGradient> + <linearGradient id="linear-gradient-2" x1="46.38" y1="94.56" x2="-29.88" y2="53" gradientUnits="userSpaceOnUse" gradientTransform="matrix(0.276923, 0, 0, 0.276923, 3.212999, 2)"> + <stop offset="0" stop-color="#005a8e"/> + <stop offset="0.54" stop-color="#1a7db6"/> + </linearGradient> + </defs> + <title>VespaLogo copy</title> + <polygon class="cls-1" points="22.736 2 8.685 11.543 8.685 27.971 22.736 18.064 36.787 27.971 36.787 11.543 22.736 2"/> + <polygon class="cls-2" points="22.736 18.064 22.736 2 36.787 11.543 36.787 27.97 22.736 18.064"/> + <polygon class="cls-3" points="17.264 21.936 3.213 12.029 3.213 28.457 17.264 38 31.315 28.457 31.315 12.029 17.264 21.936"/> + <polygon class="cls-4" points="17.264 21.936 17.264 38 3.213 28.457 3.213 12.03 17.264 21.936"/> +</svg>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/icons/document_icon.png b/integration/intellij/src/main/resources/icons/document_icon.png Binary files differindex 497c218ab91..497c218ab91 100644 --- a/sd-plugin/src/main/resources/icons/document_icon.png +++ b/integration/intellij/src/main/resources/icons/document_icon.png diff --git a/sd-plugin/src/main/resources/icons/document_summary_icon.png b/integration/intellij/src/main/resources/icons/document_summary_icon.png Binary files differindex 72f42c68902..72f42c68902 100644 --- a/sd-plugin/src/main/resources/icons/document_summary_icon.png +++ b/integration/intellij/src/main/resources/icons/document_summary_icon.png diff --git a/sd-plugin/src/main/resources/icons/first_phase_icon.png b/integration/intellij/src/main/resources/icons/first_phase_icon.png Binary files differindex 18f24e9e462..18f24e9e462 100644 --- a/sd-plugin/src/main/resources/icons/first_phase_icon.png +++ b/integration/intellij/src/main/resources/icons/first_phase_icon.png diff --git a/sd-plugin/src/main/resources/icons/imported_field_icon.png b/integration/intellij/src/main/resources/icons/imported_field_icon.png Binary files differindex 1ec9b1d6a96..1ec9b1d6a96 100644 --- a/sd-plugin/src/main/resources/icons/imported_field_icon.png +++ b/integration/intellij/src/main/resources/icons/imported_field_icon.png diff --git a/sd-plugin/src/main/resources/icons/macro_icon.png b/integration/intellij/src/main/resources/icons/macro_icon.png Binary files differindex 736d8168119..736d8168119 100644 --- a/sd-plugin/src/main/resources/icons/macro_icon.png +++ b/integration/intellij/src/main/resources/icons/macro_icon.png diff --git a/sd-plugin/src/main/resources/icons/override_macro_icon.png b/integration/intellij/src/main/resources/icons/override_macro_icon.png Binary files differindex 024eb777eef..024eb777eef 100644 --- a/sd-plugin/src/main/resources/icons/override_macro_icon.png +++ b/integration/intellij/src/main/resources/icons/override_macro_icon.png diff --git a/sd-plugin/src/main/resources/icons/sd_icon.png b/integration/intellij/src/main/resources/icons/sd_icon.png Binary files differindex 1c96aaee633..1c96aaee633 100644 --- a/sd-plugin/src/main/resources/icons/sd_icon.png +++ b/integration/intellij/src/main/resources/icons/sd_icon.png diff --git a/sd-plugin/src/main/resources/icons/struct_field_icon.png b/integration/intellij/src/main/resources/icons/struct_field_icon.png Binary files differindex 7f5074630d4..7f5074630d4 100644 --- a/sd-plugin/src/main/resources/icons/struct_field_icon.png +++ b/integration/intellij/src/main/resources/icons/struct_field_icon.png diff --git a/sd-plugin/src/main/resources/icons/struct_icon.png b/integration/intellij/src/main/resources/icons/struct_icon.png Binary files differindex c2f6fdc8440..c2f6fdc8440 100644 --- a/sd-plugin/src/main/resources/icons/struct_icon.png +++ b/integration/intellij/src/main/resources/icons/struct_icon.png diff --git a/sd-plugin/src/main/resources/icons/summary_def_icon.png b/integration/intellij/src/main/resources/icons/summary_def_icon.png Binary files differindex a8faf12601f..a8faf12601f 100644 --- a/sd-plugin/src/main/resources/icons/summary_def_icon.png +++ b/integration/intellij/src/main/resources/icons/summary_def_icon.png diff --git a/parent/pom.xml b/parent/pom.xml index eac9cf53d71..22925af4502 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -874,7 +874,7 @@ <aircompressor.version>0.21</aircompressor.version> <airline.version>0.7</airline.version> <antlr.version>3.5.2</antlr.version> - <antlr4.version>4.9.3</antlr4.version> + <antlr4.version>4.5</antlr4.version> <apache.httpclient.version>4.5.13</apache.httpclient.version> <apache.httpcore.version>4.4.13</apache.httpcore.version> <apache.httpclient5.version>5.1.1</apache.httpclient5.version> @@ -895,6 +895,7 @@ <curator.version>5.2.0</curator.version> <commons.codec.version>1.15</commons.codec.version> <commons.math3.version>3.6.1</commons.math3.version> + <gson.version>2.8.9</gson.version> <jna.version>5.9.0</jna.version> <junit.version>5.7.0</junit.version> <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> @@ -917,7 +918,7 @@ <protobuf.version>3.11.4</protobuf.version> <spifly.version>1.3.3</spifly.version> <surefire.version>2.22.2</surefire.version> - <zookeeper.client.version>3.6.3</zookeeper.client.version> + <zookeeper.client.version>3.7.0</zookeeper.client.version> <doclint>all</doclint> <test.hide>true</test.hide> diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 495522d3cf5..810f2ad2356 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -3,13 +3,13 @@ #include <vespa/document/base/testdocman.h> #include <vespa/persistence/conformancetest/conformancetest.h> #include <vespa/persistence/spi/test.h> +#include <vespa/persistence/spi/catchresult.h> #include <vespa/persistence/spi/resource_usage_listener.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/vdslib/state/state.h> #include <vespa/vdslib/state/node.h> #include <vespa/vdslib/state/nodestate.h> @@ -18,7 +18,6 @@ #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/config-stor-distribution.h> -#include <algorithm> #include <limits> #include <gtest/gtest.h> @@ -796,6 +795,40 @@ TEST_F(ConformanceTest, testRemove) EXPECT_FALSE(getResult.hasDocument()); } +TEST_F(ConformanceTest, testRemoveMulti) +{ + document::TestDocMan testDocMan; + _factory->clear(); + PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + + BucketId bucketId1(8, 0x01); + Bucket bucket1(makeSpiBucket(bucketId1)); + Context context(Priority(0), Trace::TraceLevel(0)); + spi->createBucket(bucket1, context); + + std::vector<Document::SP> docs; + for (size_t i(0); i < 30; i++) { + docs.push_back(testDocMan.createRandomDocumentAtLocation(0x01, i)); + } + + std::vector<PersistenceProvider::TimeStampAndDocumentId> ids; + for (size_t i(0); i < docs.size(); i++) { + spi->put(bucket1, Timestamp(i), docs[i], context); + if (i & 0x1) { + ids.emplace_back(Timestamp(i), docs[i]->getId()); + } + } + + auto onDone = std::make_unique<CatchResult>(); + auto future = onDone->future_result(); + spi->removeAsync(bucket1, ids, context, std::move(onDone)); + auto result = future.get(); + ASSERT_TRUE(result); + auto removeResult = dynamic_cast<spi::RemoveResult *>(result.get()); + ASSERT_TRUE(removeResult != nullptr); + EXPECT_EQ(15u, removeResult->num_removed()); +} + TEST_F(ConformanceTest, testRemoveMerge) { document::TestDocMan testDocMan; diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index c7365f39f02..9d9f31b63a3 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -471,30 +471,33 @@ DummyPersistence::updateAsync(const Bucket& bucket, Timestamp ts, DocumentUpdate } void -DummyPersistence::removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context &, OperationComplete::UP onComplete) +DummyPersistence::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, Context &, OperationComplete::UP onComplete) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; - LOG(debug, "remove(%s, %" PRIu64 ", %s)", - b.toString().c_str(), - uint64_t(t), - did.toString().c_str()); assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); - BucketContentGuard::UP bc(acquireBucketWithLock(b)); - while (!bc) { - internal_create_bucket(b); - bc = acquireBucketWithLock(b); - } - DocEntry::SP entry((*bc)->getEntry(did)); - bool foundPut(entry.get() && !entry->isRemove()); - auto remEntry = std::make_unique<DocEntry>(t, REMOVE_ENTRY, did); - - if ((*bc)->hasTimestamp(t)) { - (*bc)->eraseEntry(t); + + uint32_t numRemoves(0); + for (const TimeStampAndDocumentId & stampedId : ids) { + const DocumentId & id = stampedId.second; + Timestamp t = stampedId.first; + LOG(debug, "remove(%s, %" PRIu64 ", %s)", b.toString().c_str(), uint64_t(t), id.toString().c_str()); + + while (!bc) { + internal_create_bucket(b); + bc = acquireBucketWithLock(b); + } + DocEntry::SP entry((*bc)->getEntry(id)); + numRemoves += (entry.get() && !entry->isRemove()) ? 1 : 0; + auto remEntry = std::make_unique<DocEntry>(t, REMOVE_ENTRY, id); + + if ((*bc)->hasTimestamp(t)) { + (*bc)->eraseEntry(t); + } + (*bc)->insert(std::move(remEntry)); } - (*bc)->insert(std::move(remEntry)); bc.reset(); - onComplete->onComplete(std::make_unique<RemoveResult>(foundPut)); + onComplete->onComplete(std::make_unique<RemoveResult>(numRemoves)); } GetResult diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 4e0d088d5d6..a7a784d0479 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -159,7 +159,7 @@ public: BucketInfoResult getBucketInfo(const Bucket&) const override; GetResult get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const override; void putAsync(const Bucket&, Timestamp, DocumentSP, Context&, OperationComplete::UP) override; - void removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context&, OperationComplete::UP) override; + void removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, DocumentUpdateSP, Context&, OperationComplete::UP) override; CreateIteratorResult diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 951dbf97cff..35654240ec7 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "abstractpersistenceprovider.h" -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/document/fieldset/fieldsets.h> -#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/idestructorcallback.h> namespace storage::spi { @@ -13,7 +10,9 @@ void AbstractPersistenceProvider::removeIfFoundAsync(const Bucket& b, Timestamp timestamp, const DocumentId& id, Context& context, OperationComplete::UP onComplete) { - removeAsync(b, timestamp, id, context, std::move(onComplete)); + std::vector<TimeStampAndDocumentId> ids; + ids.emplace_back(timestamp, id); + removeAsync(b, std::move(ids), context, std::move(onComplete)); } BucketIdListResult diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index 31db08a6f4f..e6733dc4150 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -2,6 +2,7 @@ #include "persistenceprovider.h" #include "catchresult.h" +#include <vespa/document/base/documentid.h> #include <future> namespace storage::spi { @@ -44,7 +45,9 @@ RemoveResult PersistenceProvider::remove(const Bucket& bucket, Timestamp timestamp, const DocumentId & docId, Context& context) { auto catcher = std::make_unique<CatchResult>(); auto future = catcher->future_result(); - removeAsync(bucket, timestamp, docId, context, std::move(catcher)); + std::vector<TimeStampAndDocumentId> ids; + ids.emplace_back(timestamp, docId); + removeAsync(bucket, std::move(ids), context, std::move(catcher)); return dynamic_cast<const RemoveResult &>(*future.get()); } diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 269175f7d26..8c62e691daf 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -54,6 +54,7 @@ struct PersistenceProvider { using BucketSpace = document::BucketSpace; using FieldSetSP = std::shared_ptr<document::FieldSet>; + using TimeStampAndDocumentId = std::pair<Timestamp, DocumentId>; virtual ~PersistenceProvider(); @@ -168,7 +169,7 @@ struct PersistenceProvider * @param timestamp The timestamp for the new bucket entry. * @param id The ID to remove */ - virtual void removeAsync(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&, OperationComplete::UP) = 0; + virtual void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP) = 0; /** * @see remove() diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index 73838e99a18..70bd37590a1 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -142,6 +142,7 @@ public: : _numRemoved(numRemoved) { } bool wasFound() const { return _numRemoved > 0; } uint32_t num_removed() const { return _numRemoved; } + void inc_num_removed(uint32_t add) { _numRemoved += add; } private: uint32_t _numRemoved; @@ -85,6 +85,7 @@ <module>hosted-zone-api</module> <module>http-utils</module> <module>indexinglanguage</module> + <!--<module>integration/intellij</module>--> <module>jaxrs_client_utils</module> <module>jaxrs_utils</module> <module>jdisc-cloud-aws</module> diff --git a/sd-plugin/BACKLOG.md b/sd-plugin/BACKLOG.md deleted file mode 100644 index 19e185dcc6b..00000000000 --- a/sd-plugin/BACKLOG.md +++ /dev/null @@ -1,34 +0,0 @@ -### Open Issues - -1. In some cases, the grammar prefers not to enforce bad syntax, because if the parser encounters bad syntax it stops -and can't build the PSI tree. That means none of the features will work for a file like that. For example, in cases -where an element supposes to have zero-to-one occurrences, the grammar will treat it as zero-to-many. -2. In order to enable the grammar recognize some keywords as identifiers (e.g. "filter" as a field's name), the -identifier rule (named "IdentifierVal") wraps the regex (ID_REG) and the KeywordOrIdentifier rule (which contains all -the keywords in the language). -3. The implementation of the GoTo Declaration feature is not exactly the same as IntelliJ. In IntelliJ if a reference -has several declarations, after clicking "Goto Declaration" there is a little window with all the declarations to choose -from. It can be done by changing the method "multiResolve" in SdReference.java to return more than one declaration. The -problem with that is that it causes the "Find Usages" feature to not work. For now, I decided to make the plugin -"Goto Declaration" feature show only the most specific declaration by the right rank-profile scope. -4. The "Find Usages" window can group usages only under rank-profiles and document-summaries. Other usages appear -directly under the .sd file. In order to create another group type of usages' group, you'll need to create 2 classes: -one for the extension "fileStructureGroupRuleProvider" (e.g. SdRankProfileGroupingRuleProvider.java), and one for the -grouping rule itself (e.g. SdRankProfileGroupingRule.java). -Another open problem is that the navigation isn't working in the current grouping rules. It means that when clicking on -the group headline (e.g. some name of a rank-profile) the IDE doesn't "jump" to the matching declaration. -5. Goto declaration doesn't work for document's inherits. e.g. if document A inherits from document B, B doesn't have a -reference to its declaration. -6. There aren't any tests for the plugin. - -### Some useful links: -1. JetBrains official tutorials: https://plugins.jetbrains.com/docs/intellij/custom-language-support.html and -https://plugins.jetbrains.com/docs/intellij/custom-language-support-tutorial.html -2. Grammar-Kit HOWTO: Helps to understand the BNF syntax. - https://github.com/JetBrains/Grammar-Kit/blob/master/HOWTO.md -3. How to deal with left-recursion in the grammar (in SD for example it happens in expressions). Last answer here: -https://intellij-support.jetbrains.com/hc/en-us/community/posts/360001258300-What-s-the-alternative-to-left-recursion-in-GrammarKit- -4. Great tutorial for a custom-language-plugin, but only for the basics (mainly the parser and lexer): - https://medium.com/@shan1024/custom-language-plugin-development-for-intellij-idea-part-01-d6a41ab96bc9 -5. Code of Dart (some custom language) plugin for IntelliJ: -https://github.com/JetBrains/intellij-plugins/tree/0f07ca63355d5530b441ca566c98f17c560e77f8/Dart
\ No newline at end of file diff --git a/sd-plugin/README.md b/sd-plugin/README.md deleted file mode 100644 index 1cab356f78e..00000000000 --- a/sd-plugin/README.md +++ /dev/null @@ -1,42 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -This directory holds the code for an IntelliJ plugin for reading SD files. - -NOTE: This is the source code, not the plugin itself. In order to be able to use the plugin you'll need to download it -from JetBrains Marketplace or create a zip file and load it to IntelliJ by choosing "Install Plugin from Disk". - -Before starting, you should: -1. Download Gradle 7 (if you don't have it already). -2. Make sure that the bundled Plugin DevKit plugin is enabled (inside IntelliJ). -3. Optional- Download Grammar-Kit plugin from JetBrains Marketplace (inside IntelliJ). It helps with reading the .bnf file. -4. Optional- Download PsiViewer plugin from JetBrains Marketplace (inside IntelliJ). It helps to test the grammar defined -in the .bnf file. - -### Working Process -The grammar is defined in 2 files: -- sd.bnf -- sd.flex - -In order to generate the lexer and parser's code, you need to run in the command line: - - ./gradlew generateSdParser - ./gradlew generateSdLexer - -You should now have a "gen" folder next to the "java" folder, and it contains all the parser and lexer code. - -NOTE- Running those tasks would reset the "gen" folder, and all the previous generated files would be deleted before the -new ones would be generated. - -Now, you can run the gradle task "intellij/runIde" (or "./gradlew runIde" in the command line), open a project with some sd file and see how the plugin works on it. - -### Build the Plugin -In order to build the plugin and create a zip file from it, you should run the command: - - ./gradlew buildPlugin - -Or since it's a default task you can just run: - - ./gradlew - -This task also invokes the tasks generateSdParser and generateSdLexer as a part of the building process. - -Now, you'll have a zip file in the directory build\distributions. diff --git a/sd-plugin/gradle/wrapper/gradle-wrapper.jar b/sd-plugin/gradle/wrapper/gradle-wrapper.jar Binary files differdeleted file mode 100644 index 7454180f2ae..00000000000 --- a/sd-plugin/gradle/wrapper/gradle-wrapper.jar +++ /dev/null diff --git a/sd-plugin/gradle/wrapper/gradle-wrapper.properties b/sd-plugin/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 69a9715077f..00000000000 --- a/sd-plugin/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/sd-plugin/gradlew b/sd-plugin/gradlew deleted file mode 100755 index 744e882ed57..00000000000 --- a/sd-plugin/gradlew +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env sh - -# -# Copyright 2015 the original author or authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MSYS* | MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=`expr $i + 1` - done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -exec "$JAVACMD" "$@" diff --git a/sd-plugin/gradlew.bat b/sd-plugin/gradlew.bat deleted file mode 100644 index 107acd32c4e..00000000000 --- a/sd-plugin/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/sd-plugin/settings.gradle b/sd-plugin/settings.gradle deleted file mode 100644 index 5d47bd4197c..00000000000 --- a/sd-plugin/settings.gradle +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright 2000-2020 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. - -rootProject.name = 'sd_language_plugin' diff --git a/sd-plugin/src/main/resources/META-INF/plugin.xml b/sd-plugin/src/main/resources/META-INF/plugin.xml deleted file mode 100644 index dd268bec16f..00000000000 --- a/sd-plugin/src/main/resources/META-INF/plugin.xml +++ /dev/null @@ -1,53 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<idea-plugin> - <id>org.intellij.sdk.language</id> - <name>SdReader</name> - - <!-- Text to display as company information on Preferences/Settings | Plugin page --> - <vendor>Yahoo</vendor> - - <!-- Product and plugin compatibility requirements --> - <depends>com.intellij.java</depends> - <depends>com.intellij.modules.platform</depends> - - <!-- Text to display as description on Preferences/Settings | Plugin page --> - <description><![CDATA[ - <p>Support for reading Vespa's SD files.</p> - <p>Features:</p> - <ul> - <li>Structure View</li> - <li>Find Usages</li> - <li>Call Hierarchy (for functions/macros)</li> - <li>Go To Declaration</li> - <li>Go to Symbol</li> - <li>Refactoring (Only for uses of the same implementation)</li> - <li>Syntax Highlighting</li> - <li>Commenter- enable turning lines into a comment with "Code -> Comment with line comment"</li> - </ul> - <p>Find more information on Vespa's SD files <a href="https://docs.vespa.ai/en/reference/schema-reference.html">here</a>.</p> - ]]></description> - - <!-- Extension points defined by the plugin --> - <extensions defaultExtensionNs="com.intellij"> - <fileType name="Sd File" implementationClass="org.intellij.sdk.language.SdFileType" fieldName="INSTANCE" - language="Sd" extensions="sd"/> - <lang.parserDefinition language="Sd" implementationClass="org.intellij.sdk.language.parser.SdParserDefinition"/> - <lang.syntaxHighlighterFactory language="Sd" implementationClass="org.intellij.sdk.language.SdSyntaxHighlighterFactory"/> - <completion.contributor language="Sd" implementationClass="org.intellij.sdk.language.SdCompletionContributor"/> - - <lang.findUsagesProvider language="Sd" implementationClass="org.intellij.sdk.language.findUsages.SdFindUsagesProvider"/> - <findUsagesHandlerFactory implementation="org.intellij.sdk.language.findUsages.SdFindUsagesHandlerFactory"/> - <fileStructureGroupRuleProvider implementation="org.intellij.sdk.language.findUsages.SdRankProfileGroupingRuleProvider"/> - <fileStructureGroupRuleProvider implementation="org.intellij.sdk.language.findUsages.SdDocumentSummaryGroupingRuleProvider"/> - <elementDescriptionProvider implementation="org.intellij.sdk.language.psi.SdElementDescriptionProvider"/> - - <lang.psiStructureViewFactory language="Sd" implementationClass="org.intellij.sdk.language.structure.SdStructureViewFactory"/> - <codeStyleSettingsProvider implementation="org.intellij.sdk.language.SdCodeStyleSettingsProvider"/> - <langCodeStyleSettingsProvider implementation="org.intellij.sdk.language.SdLanguageCodeStyleSettingsProvider"/> - <lang.commenter language="Sd" implementationClass="org.intellij.sdk.language.SdCommenter"/> - <lang.refactoringSupport language="Sd" implementationClass="org.intellij.sdk.language.SdRefactoringSupportProvider"/> - <gotoSymbolContributor implementation="org.intellij.sdk.language.SdChooseByNameContributor"/> - <callHierarchyProvider language="Sd" implementationClass="org.intellij.sdk.language.hierarchy.SdCallHierarchyProvider"/> - </extensions> - -</idea-plugin>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/META-INF/pluginIcon.svg b/sd-plugin/src/main/resources/META-INF/pluginIcon.svg deleted file mode 100644 index caacd6a053e..00000000000 --- a/sd-plugin/src/main/resources/META-INF/pluginIcon.svg +++ /dev/null @@ -1,10 +0,0 @@ -<svg version="1.2" baseProfile="tiny-ps" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32"> - <title>New Project</title> - <defs> - <image width="30" height="29" id="img1" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAAdCAMAAACKeiw+AAAAAXNSR0IB2cksfwAAAUpQTFRFAAAAHKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0lOzY/wAAAG50Uk5TAA5Acarj/8dVMgdq1LgjXdzxzbAfO/iV1epHHGO/f9hcFZ3lQiR08oAPeL7CuWkIYKieoI8u9/n69en2/eAX6GY6rljt7MzmWv4ac5jwEO5RM7381k8skCnX3srvyDdB6/N8jNLnBfT7wXqSr7W47lQGAAABpElEQVR4nIVT6z8CQRS9+7qVNk3FKiqUIvJWbSLvjQh5hkgeef//X83s2rb6hfNhZvecmXvP3LkD8AOOF0QJKWyi3cFBO3ocThmbcDl7e1pVzk08Xp+9j333Cy6ieN2t8oA/IA42Iw4FQ+gPW2oIh0fa0o1GomjqnBujYx1mYvFxv9vIP+jFCejEaAi9vfpm0eOkexOTU8npmVmYm19YXEpRA6LCaHAQ4qDTUlrJqFkeJGU5t7KaBwiii/F29LEka+r6xmZ6C7Z3+uJaYZdG3UM7QFEMCCyHpwCQ2s9Tmf6sanQ4QLEIMRtJMTlbCh+yeYfJyhEdjtEWgzJqutUT9fTs3JRz24xCLDdlmCmkoxe/ywCXauQvGVQJDGuWXJGudGvXN9RR1ZAT8i1l7lCqNA+WVmoJtQRwX394fHrmzYM1BKMs2mlpLeMDkF+ymVeS18siNAD4gF5UOMnW3yj7/vGZ/KoBK6rMW1fSCfNKIEi6X6gR9L92AE7u1kwDFiNjZytiqGX1P40MXFAMYPRKoyBVJC2hzAV/PSKGRrFSpogVGxb3De/eO/7N6EMHAAAAAElFTkSuQmCC"/> - </defs> - <style> - tspan { white-space:pre } - </style> - <use id="Layer 2" href="#img1" x="1" y="2" /> -</svg>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg b/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg deleted file mode 100644 index caacd6a053e..00000000000 --- a/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg +++ /dev/null @@ -1,10 +0,0 @@ -<svg version="1.2" baseProfile="tiny-ps" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32"> - <title>New Project</title> - <defs> - <image width="30" height="29" id="img1" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAAdCAMAAACKeiw+AAAAAXNSR0IB2cksfwAAAUpQTFRFAAAAHKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0lOzY/wAAAG50Uk5TAA5Acarj/8dVMgdq1LgjXdzxzbAfO/iV1epHHGO/f9hcFZ3lQiR08oAPeL7CuWkIYKieoI8u9/n69en2/eAX6GY6rljt7MzmWv4ac5jwEO5RM7381k8skCnX3srvyDdB6/N8jNLnBfT7wXqSr7W47lQGAAABpElEQVR4nIVT6z8CQRS9+7qVNk3FKiqUIvJWbSLvjQh5hkgeef//X83s2rb6hfNhZvecmXvP3LkD8AOOF0QJKWyi3cFBO3ocThmbcDl7e1pVzk08Xp+9j333Cy6ieN2t8oA/IA42Iw4FQ+gPW2oIh0fa0o1GomjqnBujYx1mYvFxv9vIP+jFCejEaAi9vfpm0eOkexOTU8npmVmYm19YXEpRA6LCaHAQ4qDTUlrJqFkeJGU5t7KaBwiii/F29LEka+r6xmZ6C7Z3+uJaYZdG3UM7QFEMCCyHpwCQ2s9Tmf6sanQ4QLEIMRtJMTlbCh+yeYfJyhEdjtEWgzJqutUT9fTs3JRz24xCLDdlmCmkoxe/ywCXauQvGVQJDGuWXJGudGvXN9RR1ZAT8i1l7lCqNA+WVmoJtQRwX394fHrmzYM1BKMs2mlpLeMDkF+ymVeS18siNAD4gF5UOMnW3yj7/vGZ/KoBK6rMW1fSCfNKIEi6X6gR9L92AE7u1kwDFiNjZytiqGX1P40MXFAMYPRKoyBVJC2hzAV/PSKGRrFSpogVGxb3De/eO/7N6EMHAAAAAElFTkSuQmCC"/> - </defs> - <style> - tspan { white-space:pre } - </style> - <use id="Layer 2" href="#img1" x="1" y="2" /> -</svg>
\ No newline at end of file diff --git a/sd-plugin/src/main/resources/flex/idea-flex.skeleton b/sd-plugin/src/main/resources/flex/idea-flex.skeleton deleted file mode 100644 index 218399976ca..00000000000 --- a/sd-plugin/src/main/resources/flex/idea-flex.skeleton +++ /dev/null @@ -1,248 +0,0 @@ - - /** This character denotes the end of file */ - public static final int YYEOF = -1; - - /** initial size of the lookahead buffer */ ---- private static final int ZZ_BUFFERSIZE = ...; - - /** lexical states */ ---- lexical states, charmap - - /* error codes */ - private static final int ZZ_UNKNOWN_ERROR = 0; - private static final int ZZ_NO_MATCH = 1; - private static final int ZZ_PUSHBACK_2BIG = 2; - - /* error messages for the codes above */ - private static final String[] ZZ_ERROR_MSG = { - "Unknown internal scanner error", - "Error: could not match input", - "Error: pushback value was too large" - }; - ---- isFinal list - /** the input device */ - private java.io.Reader zzReader; - - /** the current state of the DFA */ - private int zzState; - - /** the current lexical state */ - private int zzLexicalState = YYINITIAL; - - /** this buffer contains the current text to be matched and is - the source of the yytext() string */ - private CharSequence zzBuffer = ""; - - /** the textposition at the last accepting state */ - private int zzMarkedPos; - - /** the current text position in the buffer */ - private int zzCurrentPos; - - /** startRead marks the beginning of the yytext() string in the buffer */ - private int zzStartRead; - - /** endRead marks the last character in the buffer, that has been read - from input */ - private int zzEndRead; - - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ - private boolean zzAtBOL = true; - - /** zzAtEOF == true <=> the scanner is at the EOF */ - private boolean zzAtEOF; - - /** denotes if the user-EOF-code has already been executed */ - private boolean zzEOFDone; - ---- user class code - ---- constructor declaration - - public final int getTokenStart() { - return zzStartRead; - } - - public final int getTokenEnd() { - return getTokenStart() + yylength(); - } - - public void reset(CharSequence buffer, int start, int end, int initialState) { - zzBuffer = buffer; - zzCurrentPos = zzMarkedPos = zzStartRead = start; - zzAtEOF = false; - zzAtBOL = true; - zzEndRead = end; - yybegin(initialState); - } - - /** - * Refills the input buffer. - * - * @return {@code false}, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ - private boolean zzRefill() throws java.io.IOException { - return true; - } - - - /** - * Returns the current lexical state. - */ - public final int yystate() { - return zzLexicalState; - } - - - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ - public final void yybegin(int newState) { - zzLexicalState = newState; - } - - - /** - * Returns the text matched by the current regular expression. - */ - public final CharSequence yytext() { - return zzBuffer.subSequence(zzStartRead, zzMarkedPos); - } - - - /** - * Returns the character at position {@code pos} from the - * matched text. - * - * It is equivalent to yytext().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to yylength()-1. - * - * @return the character at position pos - */ - public final char yycharat(int pos) { - return zzBuffer.charAt(zzStartRead+pos); - } - - - /** - * Returns the length of the matched text region. - */ - public final int yylength() { - return zzMarkedPos-zzStartRead; - } - - - /** - * Reports an error that occurred while scanning. - * - * In a wellformed scanner (no or only correct usage of - * yypushback(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ ---- zzScanError declaration - String message; - try { - message = ZZ_ERROR_MSG[errorCode]; - } - catch (ArrayIndexOutOfBoundsException e) { - message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; - } - ---- throws clause - } - - - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than yylength()! - */ ---- yypushback decl (contains zzScanError exception) - if ( number > yylength() ) - zzScanError(ZZ_PUSHBACK_2BIG); - - zzMarkedPos -= number; - } - - ---- zzDoEOF - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ ---- yylex declaration - int zzInput; - int zzAction; - - // cached fields: - int zzCurrentPosL; - int zzMarkedPosL; - int zzEndReadL = zzEndRead; - CharSequence zzBufferL = zzBuffer; - ---- local declarations - - while (true) { - zzMarkedPosL = zzMarkedPos; - ---- start admin (line, char, col count) - zzAction = -1; - - zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; - ---- start admin (lexstate etc) - - zzForAction: { - while (true) { - ---- next input, line, col, char count, next transition, isFinal action - zzAction = zzState; - zzMarkedPosL = zzCurrentPosL; ---- line count update - } - - } - } - - // store back cached position - zzMarkedPos = zzMarkedPosL; ---- char count update - - if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { - zzAtEOF = true; ---- eofvalue - } - else { ---- actions - default: ---- no match - } - } - } - } - ---- main - -} diff --git a/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar b/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar Binary files differdeleted file mode 100644 index c152c2c3384..00000000000 --- a/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar +++ /dev/null diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index d4ad85d88d9..3397b424ed0 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -28,7 +28,6 @@ #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> -#include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/searchlib/index/docbuilder.h> @@ -40,7 +39,6 @@ #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/exceptions.h> @@ -165,12 +163,10 @@ public: AttributeVector::SP addAttribute(const AttributeSpec &spec) { auto ret = _mgr->addAttribute(spec.getName(), AttributeFactory::createAttribute(spec.getName(), spec.getConfig())); - allocAttributeWriter(); return ret; } void add_attribute(AttributeVector::SP attr) { _mgr->addAttribute(attr->getName(), std::move(attr)); - allocAttributeWriter(); } void put(SerialNum serialNum, const Document &doc, DocumentIdT lid) { _aw->put(serialNum, doc, lid, emptyCallback); @@ -225,6 +221,7 @@ TEST_F(AttributeWriterTest, handles_put) auto a2 = addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); auto a3 = addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}); auto a4 = addAttribute({"a4", AVConfig(AVBasicType::STRING)}); + allocAttributeWriter(); attribute::IntegerContent ibuf; attribute::FloatContent fbuf; @@ -303,6 +300,7 @@ TEST_F(AttributeWriterTest, handles_predicate_put) DocBuilder idb(s); auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); + allocAttributeWriter(); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); @@ -349,6 +347,7 @@ TEST_F(AttributeWriterTest, handles_remove) { auto a1 = addAttribute("a1"); auto a2 = addAttribute("a2"); + allocAttributeWriter(); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); @@ -371,6 +370,7 @@ TEST_F(AttributeWriterTest, handles_batch_remove) { auto a1 = addAttribute("a1"); auto a2 = addAttribute("a2"); + allocAttributeWriter(); fillAttribute(a1, 4, 22, 1); fillAttribute(a2, 4, 33, 1); @@ -397,6 +397,7 @@ verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::string TEST_F(AttributeWriterTest, visibility_delay_is_honoured) { auto a1 = addAttribute({"a1", AVConfig(AVBasicType::STRING)}); + allocAttributeWriter(); Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::STRING, CollectionType::SINGLE)); DocBuilder idb(s); @@ -442,6 +443,7 @@ TEST_F(AttributeWriterTest, visibility_delay_is_honoured) TEST_F(AttributeWriterTest, handles_predicate_remove) { auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); + allocAttributeWriter(); Schema s; s.addAttributeField( Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); @@ -465,6 +467,7 @@ TEST_F(AttributeWriterTest, handles_update) { auto a1 = addAttribute("a1"); auto a2 = addAttribute("a2"); + allocAttributeWriter(); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); @@ -504,6 +507,7 @@ TEST_F(AttributeWriterTest, handles_update) TEST_F(AttributeWriterTest, handles_predicate_update) { auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); + allocAttributeWriter(); Schema schema; schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); @@ -680,6 +684,7 @@ createTensorPutDoc(DocBuilder &builder, const Value &tensor) { TEST_F(AttributeWriterTest, can_write_to_tensor_attribute) { auto a1 = createTensorAttribute(*this); + allocAttributeWriter(); Schema s = createTensorSchema(); DocBuilder builder(s); auto tensor = make_tensor(TensorSpec(sparse_tensor) @@ -697,6 +702,7 @@ TEST_F(AttributeWriterTest, can_write_to_tensor_attribute) TEST_F(AttributeWriterTest, handles_tensor_assign_update) { auto a1 = createTensorAttribute(*this); + allocAttributeWriter(); Schema s = createTensorSchema(); DocBuilder builder(s); auto tensor = make_tensor(TensorSpec(sparse_tensor) @@ -762,6 +768,7 @@ putAttributes(AttributeWriterTest &t, std::vector<uint32_t> expExecuteHistory) auto a1 = t.addAttribute(a1_name); auto a2 = t.addAttribute(a2_name); auto a3 = t.addAttribute(a3_name); + t.allocAttributeWriter(); EXPECT_EQ(1u, a1->getNumDocs()); EXPECT_EQ(1u, a2->getNumDocs()); @@ -899,6 +906,7 @@ public: tensor = make_tensor(TensorSpec(dense_tensor) .add({{"x", 0}}, 3).add({{"x", 1}}, 5)); attr->exp_tensor = tensor.get(); + allocAttributeWriter(); } void expect_tensor_attr_calls(size_t exp_prepare_cnt, size_t exp_complete_cnt, @@ -1028,6 +1036,7 @@ TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribut TEST_F(AttributeWriterTest, ignores_force_commit_serial_not_greater_than_create_serial) { auto a1 = addAttribute("a1"); + allocAttributeWriter(); a1->setCreateSerialNum(100); EXPECT_EQ(0u, test_force_commit(*a1, 50u)); EXPECT_EQ(0u, test_force_commit(*a1, 100u)); @@ -1112,6 +1121,7 @@ StructArrayWriterTest::~StructArrayWriterTest() = default; TEST_F(StructArrayWriterTest, update_with_doc_argument_updates_struct_field_attributes) { + allocAttributeWriter(); auto doc = makeDoc(10, {11, 12}); put(10, *doc, 1); checkAttrs(1, 10, {11, 12}); @@ -1169,6 +1179,7 @@ public: TEST_F(StructMapWriterTest, update_with_doc_argument_updates_struct_field_attributes) { + allocAttributeWriter(); auto doc = makeDoc(10, {{1, 11}, {2, 12}}); put(10, *doc, 1); checkAttrs(1, 10, {{1, 11}, {2, 12}}); diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index b2acc8703f3..cd694cb938c 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -287,7 +287,7 @@ struct FastAccessFixture vespalib::mkdir(BASE_DIR); } ~FastAccessFixture() { - _writeService.sync_all_executors(); + _writeService.shutdown(); } }; @@ -377,18 +377,12 @@ struct FeedViewComparer void expect_equal_index_adapter() { EXPECT_EQUAL(_old->getIndexWriter().get(), _new->getIndexWriter().get()); } - void expect_equal_attribute_writer() { - EXPECT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get()); - } void expect_not_equal_attribute_writer() { EXPECT_NOT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get()); } void expect_equal_summary_adapter() { EXPECT_EQUAL(_old->getSummaryAdapter().get(), _new->getSummaryAdapter().get()); } - void expect_equal_schema() { - EXPECT_EQUAL(_old->getSchema().get(), _new->getSchema().get()); - } void expect_not_equal_schema() { EXPECT_NOT_EQUAL(_old->getSchema().get(), _new->getSchema().get()); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 091292b3151..e3e6ac6321e 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -34,6 +34,7 @@ #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/destructor_callbacks.h> using namespace cloud::config::filedistribution; using namespace document; @@ -65,7 +66,6 @@ typedef StoreOnlyDocSubDB::Config StoreOnlyConfig; typedef StoreOnlyDocSubDB::Context StoreOnlyContext; typedef FastAccessDocSubDB::Config FastAccessConfig; typedef FastAccessDocSubDB::Context FastAccessContext; -typedef SearchableDocSubDB::Config SearchableConfig; typedef SearchableDocSubDB::Context SearchableContext; typedef std::vector<AttributeGuard> AttributeGuardList; @@ -203,9 +203,9 @@ MyFastAccessContext::~MyFastAccessContext() = default; struct MySearchableConfig { - SearchableConfig _cfg; + FastAccessConfig _cfg; MySearchableConfig() - : _cfg(MyFastAccessConfig<false>()._cfg, 1) + : _cfg(MyFastAccessConfig<false>()._cfg) { } }; @@ -313,9 +313,17 @@ struct FixtureBase init(); } ~FixtureBase() { - _writeService.sync_all_executors(); - _writeService.master().execute(makeLambdaTask([this]() { _subDb.close(); })); - _writeService.sync_all_executors(); + _writeService.master().execute(makeLambdaTask([this]() { _subDb.close(); })); + _writeService.shutdown(); + } + void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc) { + vespalib::Gate gate; + _subDb.setBucketStateCalculator(calc, std::make_shared<vespalib::GateCallback>(gate)); + gate.await(); + } + template <typename FunctionType> + void runInMasterAndSync(FunctionType func) { + proton::test::runInMasterAndSync(_writeService, func); } template <typename FunctionType> void runInMaster(FunctionType func) { @@ -328,16 +336,16 @@ struct FixtureBase initializer::TaskRunner taskRunner(executor); taskRunner.runTask(task); auto sessionMgr = std::make_shared<SessionManager>(1); - runInMaster([&] () { _subDb.initViews(*_snapshot->_cfg, sessionMgr); }); + runInMasterAndSync([&]() { _subDb.initViews(*_snapshot->_cfg, sessionMgr); }); } void basicReconfig(SerialNum serialNum) { - runInMaster([&] () { performReconfig(serialNum, TwoAttrSchema(), ConfigDir2::dir()); }); + runInMasterAndSync([&]() { performReconfig(serialNum, TwoAttrSchema(), ConfigDir2::dir()); }); } void reconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) { - runInMaster([&] () { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); }); + runInMasterAndSync([&]() { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); }); } void performReconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) { - MyConfigSnapshot::UP newCfg(new MyConfigSnapshot(reconfigSchema, reconfigConfigDir)); + auto newCfg = std::make_unique<MyConfigSnapshot>(reconfigSchema, reconfigConfigDir); DocumentDBConfig::ComparisonResult cmpResult; cmpResult.attributesChanged = true; cmpResult.documenttypesChanged = true; @@ -353,9 +361,7 @@ struct FixtureBase } _subDb.onReprocessDone(serialNum); } - void sync() { - _writeService.master().sync(); - } + proton::IAttributeManager::SP getAttributeManager() { return _subDb.getAttributeManager(); } @@ -386,20 +392,20 @@ typedef BaseTraitsT<OneAttrSchema, ConfigDir1> BaseTraits; struct StoreOnlyTraits : public BaseTraits { - typedef MyStoreOnlyConfig Config; - typedef MyStoreOnlyContext Context; - typedef StoreOnlyDocSubDB SubDB; - typedef StoreOnlyFeedView FeedView; + using Config = MyStoreOnlyConfig; + using Context = MyStoreOnlyContext; + using SubDB = StoreOnlyDocSubDB; + using FeedView = StoreOnlyFeedView; }; typedef FixtureBase<StoreOnlyTraits> StoreOnlyFixture; struct FastAccessTraits : public BaseTraits { - typedef MyFastAccessConfig<false> Config; - typedef MyFastAccessContext Context; - typedef FastAccessDocSubDB SubDB; - typedef FastAccessFeedView FeedView; + using Config = MyFastAccessConfig<false>; + using Context = MyFastAccessContext; + using SubDB = FastAccessDocSubDB; + using FeedView = FastAccessFeedView; }; typedef FixtureBase<FastAccessTraits> FastAccessFixture; @@ -407,10 +413,10 @@ typedef FixtureBase<FastAccessTraits> FastAccessFixture; template <typename ConfigDirT> struct FastAccessOnlyTraitsBase : public BaseTraitsT<TwoAttrSchema, ConfigDirT> { - typedef MyFastAccessConfig<true> Config; - typedef MyFastAccessContext Context; - typedef FastAccessDocSubDB SubDB; - typedef FastAccessFeedView FeedView; + using Config = MyFastAccessConfig<true>; + using Context = MyFastAccessContext; + using SubDB = FastAccessDocSubDB; + using FeedView = FastAccessFeedView; }; // Setup with 1 fast-access attribute @@ -420,10 +426,10 @@ typedef FixtureBase<FastAccessOnlyTraits> FastAccessOnlyFixture; template <typename SchemaT, typename ConfigDirT> struct SearchableTraitsBase : public BaseTraitsT<SchemaT, ConfigDirT> { - typedef MySearchableConfig Config; - typedef MySearchableContext Context; - typedef SearchableDocSubDB SubDB; - typedef proton::SearchableFeedView FeedView; + using Config = MySearchableConfig; + using Context = MySearchableContext; + using SubDB = SearchableDocSubDB; + using FeedView = proton::SearchableFeedView; }; typedef SearchableTraitsBase<OneAttrSchema, ConfigDir1> SearchableTraits; @@ -567,14 +573,14 @@ TEST_F("require that subdb reflect retirement", FastAccessFixture) auto calc = std::make_shared<proton::test::BucketStateCalculator>(); calc->setNodeRetired(true); - f._subDb.setBucketStateCalculator(calc); + f.setBucketStateCalculator(calc); EXPECT_TRUE(f._subDb.isNodeRetired()); auto retired_cfg = f._subDb.computeCompactionStrategy(cfg); EXPECT_TRUE(cfg != retired_cfg); EXPECT_TRUE(search::CompactionStrategy(0.5, 0.5) == retired_cfg); calc->setNodeRetired(false); - f._subDb.setBucketStateCalculator(calc); + f.setBucketStateCalculator(calc); EXPECT_FALSE(f._subDb.isNodeRetired()); unretired_cfg = f._subDb.computeCompactionStrategy(cfg); EXPECT_TRUE(cfg == unretired_cfg); @@ -590,21 +596,18 @@ TEST_F("require that attribute compaction config reflect retirement", FastAccess auto calc = std::make_shared<proton::test::BucketStateCalculator>(); calc->setNodeRetired(true); - f._subDb.setBucketStateCalculator(calc); - f._writeService.sync_all_executors(); + f.setBucketStateCalculator(calc); guard = f._subDb.getAttributeManager()->getAttribute("attr1"); EXPECT_EQUAL(retired_cfg, (*guard)->getConfig().getCompactionStrategy()); EXPECT_EQUAL(retired_cfg, dynamic_cast<const proton::DocumentMetaStore &>(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy()); f.basicReconfig(10); - f._writeService.sync_all_executors(); guard = f._subDb.getAttributeManager()->getAttribute("attr1"); EXPECT_EQUAL(retired_cfg, (*guard)->getConfig().getCompactionStrategy()); EXPECT_EQUAL(retired_cfg, dynamic_cast<const proton::DocumentMetaStore &>(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy()); calc->setNodeRetired(false); - f._subDb.setBucketStateCalculator(calc); - f._writeService.sync_all_executors(); + f.setBucketStateCalculator(calc); guard = f._subDb.getAttributeManager()->getAttribute("attr1"); EXPECT_EQUAL(default_cfg, (*guard)->getConfig().getCompactionStrategy()); EXPECT_EQUAL(default_cfg, dynamic_cast<const proton::DocumentMetaStore &>(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy()); @@ -783,27 +786,34 @@ struct DocumentHandler } void putDoc(PutOperation &op) { IFeedView::SP feedView = _f._subDb.getFeedView(); + vespalib::Gate gate; _f.runInMaster([&]() { feedView->preparePut(op); feedView->handlePut(FeedToken(), op); - feedView->forceCommit(op.getSerialNum()); - } ); + feedView->forceCommit(CommitParam(op.getSerialNum()), std::make_shared<vespalib::GateCallback>(gate)); + }); + gate.await(); } void moveDoc(MoveOperation &op) { IFeedView::SP feedView = _f._subDb.getFeedView(); + vespalib::Gate gate; _f.runInMaster([&]() { - feedView->handleMove(op, IDestructorCallback::SP()); - feedView->forceCommit(op.getSerialNum()); - } ); + auto onDone = std::make_shared<vespalib::GateCallback>(gate); + feedView->handleMove(op, onDone); + feedView->forceCommit(CommitParam(op.getSerialNum()), onDone); + }); + gate.await(); } void removeDoc(RemoveOperation &op) { IFeedView::SP feedView = _f._subDb.getFeedView(); + vespalib::Gate gate; _f.runInMaster([&]() { feedView->prepareRemove(op); feedView->handleRemove(FeedToken(), op); - feedView->forceCommit(op.getSerialNum()); - } ); + feedView->forceCommit(CommitParam(op.getSerialNum()), std::make_shared<vespalib::GateCallback>(gate)); + }); + gate.await(); } void putDocs() { PutOperation putOp = createPut(std::move(createDoc(1, 22, 33)), Timestamp(10), 10); diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 977c899ab11..bffedfd8dab 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -185,13 +185,13 @@ struct MyFeedView : public test::DummyFeedView { int remove_count; int move_count; int prune_removed_count; + int update_count; SerialNum update_serial; const DocumentType *documentType; MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr, const DocTypeName &docTypeName); ~MyFeedView() override; - void resetPutLatch(uint32_t count) { putLatch = std::make_unique<vespalib::CountDownLatch>(count); } void preparePut(PutOperation &op) override { prepareDocumentOperation(op, op.getDocument()->getId().getGlobalId()); } @@ -389,16 +389,6 @@ FeedTokenContext::FeedTokenContext() FeedTokenContext::~FeedTokenContext() = default; -struct PutContext { - FeedTokenContext tokenCtx; - DocumentContext docCtx; - typedef std::shared_ptr<PutContext> SP; - PutContext(const vespalib::string &docId, DocBuilder &builder) : - tokenCtx(), - docCtx(docId, builder) - {} -}; - struct MyTlsWriter : TlsWriter { int store_count; int erase_count; @@ -455,40 +445,18 @@ struct FeedHandlerFixture } ~FeedHandlerFixture() { - writeService.sync_all_executors(); + writeService.shutdown(); } template <class FunctionType> inline void runAsMaster(FunctionType &&function) { writeService.master().execute(makeLambdaTask(std::move(function))); - writeService.master().sync(); + syncMaster(); } void syncMaster() { writeService.master().sync(); } }; - -struct MyConfigStore : ConfigStore { - SerialNum getBestSerialNum() const override { return 1; } - SerialNum getOldestSerialNum() const override { return 1; } - void saveConfig(const DocumentDBConfig &, SerialNum) override {} - void loadConfig(const DocumentDBConfig &, SerialNum, DocumentDBConfig::SP &) override {} - void removeInvalid() override {} - void prune(SerialNum) override {} - bool hasValidSerial(SerialNum) const override { return true; } - SerialNum getPrevValidSerial(SerialNum) const override { return 1; } - void serializeConfig(SerialNum, vespalib::nbostream &) override {} - void deserializeConfig(SerialNum, vespalib::nbostream &) override {} - void setProtonConfig(const ProtonConfigSP &) override { } -}; - - -struct ReplayTransactionLogContext { - MyConfigStore config_store; - DocumentDBConfig::SP cfgSnap; -}; - - TEST_F("require that heartBeat calls FeedView's heartBeat", FeedHandlerFixture) { diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index c35cd49e498..a030ce1c455 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -22,6 +22,7 @@ #include <vespa/searchcore/proton/test/threading_service_observer.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/log/log.h> @@ -375,6 +376,9 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; _tracer.traceCommit(attributeAdapterTypeName, param.lastSerialNum()); } + void drain(OnWriteDoneType onDone) override { + (void) onDone; + } void onReplayDone(uint32_t ) override { } bool hasStructFieldAttribute() const override { return false; } @@ -415,16 +419,6 @@ struct MyTransport : public feedtoken::ITransport MyTransport::MyTransport(MyTracer &tracer) : lastResult(), _gate(), _tracer(tracer) {} MyTransport::~MyTransport() = default; -struct MyResultHandler : public IGenericResultHandler -{ - vespalib::Gate _gate; - MyResultHandler() : _gate() {} - void handle(const storage::spi::Result &) override { - _gate.countDown(); - } - void await() { _gate.await(); } -}; - struct SchemaContext { Schema::SP _schema; @@ -447,7 +441,6 @@ SchemaContext::SchemaContext() : } SchemaContext::~SchemaContext() = default; - struct DocumentContext { Document::SP doc; @@ -511,18 +504,6 @@ struct FixtureBase virtual ~FixtureBase(); - void syncMaster() { - _writeService.master().sync(); - } - - void syncIndex() { - _writeService.sync_all_executors(); - } - - void sync() { - _writeServiceReal.sync_all_executors(); - } - const test::DocumentMetaStoreObserver &metaStoreObserver() { return _dmsc->getObserver(); } @@ -532,6 +513,10 @@ struct FixtureBase } template <typename FunctionType> + void runInMasterAndSyncAll(FunctionType func) { + test::runInMasterAndSyncAll(_writeService, func); + } + template <typename FunctionType> void runInMaster(FunctionType func) { test::runInMaster(_writeService, func); } @@ -578,7 +563,7 @@ struct FixtureBase void putAndWait(const DocumentContext &docCtx) { FeedTokenContext token(_tracer); PutOperation op(docCtx.bid, docCtx.ts, docCtx.doc); - runInMaster([this, ft=std::move(token.ft), &op] () mutable { performPut(std::move(ft), op); }); + runInMaster([this, ft = std::move(token.ft), &op]() mutable { performPut(std::move(ft), op); }); token.mt.await(); } @@ -591,7 +576,7 @@ struct FixtureBase void updateAndWait(const DocumentContext &docCtx) { FeedTokenContext token(_tracer); UpdateOperation op(docCtx.bid, docCtx.ts, docCtx.upd); - runInMaster([this, ft=std::move(token.ft), &op] () mutable { performUpdate(std::move(ft), op); }); + runInMaster([this, ft = std::move(token.ft), &op]() mutable { performUpdate(std::move(ft), op); }); token.mt.await(); } @@ -606,7 +591,7 @@ struct FixtureBase void removeAndWait(const DocumentContext &docCtx) { FeedTokenContext token(_tracer); RemoveOperationWithDocId op(docCtx.bid, docCtx.ts, docCtx.doc->getId()); - runInMaster([this, ft=std::move(token.ft), &op] () mutable { performRemove(std::move(ft), op); }); + runInMaster([this, ft = std::move(token.ft), &op]() mutable { performRemove(std::move(ft), op); }); token.mt.await(); } @@ -616,15 +601,17 @@ struct FixtureBase } } - void performMove(MoveOperation &op) { + void performMove(MoveOperation &op, IDestructorCallback::SP onDone) { op.setSerialNum(++serial); - getFeedView().handleMove(op, IDestructorCallback::SP()); + getFeedView().handleMove(op, std::move(onDone)); } void moveAndWait(const DocumentContext &docCtx, uint32_t fromLid, uint32_t toLid) { MoveOperation op(docCtx.bid, docCtx.ts, docCtx.doc, DbDocumentId(pc._params._subDbId, fromLid), pc._params._subDbId); op.setTargetLid(toLid); - runInMaster([&]() { performMove(op); }); + vespalib::Gate gate; + runInMaster([&, onDone=std::make_shared<vespalib::GateCallback>(gate)]() { performMove(op, std::move(onDone)); }); + gate.await(); } void performDeleteBucket(DeleteBucketOperation &op) { @@ -635,7 +622,7 @@ struct FixtureBase void performForceCommit() { getFeedView().forceCommit(serial); } void forceCommitAndWait() { - runInMaster([&]() { performForceCommit(); }); + runInMasterAndSyncAll([&]() { performForceCommit(); }); } bool assertTrace(const vespalib::string &exp) { @@ -662,7 +649,7 @@ struct FixtureBase fv.handleCompactLidSpace(op); } void compactLidSpaceAndWait(uint32_t wantedLidLimit) { - runInMaster([&] () { performCompactLidSpace(wantedLidLimit); }); + runInMasterAndSyncAll([&]() { performCompactLidSpace(wantedLidLimit); }); } void assertChangeHandler(document::GlobalId expGid, uint32_t expLid, uint32_t expChanges) { _gidToLidChangeHandler->assertChanges(expGid, expLid, expChanges); @@ -674,6 +661,8 @@ struct FixtureBase _gidToLidChangeHandler->assertLid(gid, expLid); } void populateBeforeCompactLidSpace(); + + void dms_commit() { _dmsc->get().commit(search::CommitParam(serial)); } }; @@ -701,7 +690,7 @@ FixtureBase::FixtureBase() } FixtureBase::~FixtureBase() { - _writeServiceReal.sync_all_executors(); + _writeServiceReal.shutdown(); } void @@ -789,6 +778,7 @@ TEST_F("require that put() updates document meta store with bucket info", { DocumentContext dc = f.doc1(); f.putAndWait(dc); + f.dms_commit(); assertBucketInfo(dc.bid, dc.ts, 1, f.getMetaStore()); // TODO: rewrite to use getBucketInfo() when available @@ -828,6 +818,7 @@ TEST_F("require that update() updates document meta store with bucket info", f.putAndWait(dc1); BucketChecksum bcs = f.getBucketDB()->get(dc1.bid).getChecksum(); f.updateAndWait(dc2); + f.dms_commit(); assertBucketInfo(dc1.bid, Timestamp(20), 1, f.getMetaStore()); // TODO: rewrite to use getBucketInfo() when available @@ -857,6 +848,7 @@ TEST_F("require that remove() updates document meta store with bucket info", f.putAndWait(dc2); BucketChecksum bcs2 = f.getBucketDB()->get(dc2.bid).getChecksum(); f.removeAndWait(DocumentContext("id:test:searchdocument:n=1:2", 20, f.getBuilder())); + f.dms_commit(); assertBucketInfo(dc1.bid, Timestamp(10), 1, f.getMetaStore()); EXPECT_FALSE(f.getMetaStore().validLid(2)); // don't remember remove @@ -934,6 +926,7 @@ TEST_F("require that handleDeleteBucket() removes documents", SearchableFeedView TEST_DO(f.assertChangeNotified(docs[2].gid(), 3)); TEST_DO(f.assertChangeNotified(docs[3].gid(), 4)); TEST_DO(f.assertChangeNotified(docs[4].gid(), 5)); + f.dms_commit(); DocumentIdT lid; EXPECT_TRUE(f.getMetaStore().getLid(docs[0].doc->getId().getGlobalId(), lid)); @@ -945,7 +938,8 @@ TEST_F("require that handleDeleteBucket() removes documents", SearchableFeedView // delete bucket for user 1 DeleteBucketOperation op(docs[0].bid); - f.runInMaster([&] () { f.performDeleteBucket(op); }); + f.runInMasterAndSyncAll([&]() { f.performDeleteBucket(op); }); + f.dms_commit(); EXPECT_EQUAL(0u, f.getBucketDB()->get(docs[0].bid).getDocumentCount()); EXPECT_EQUAL(2u, f.getBucketDB()->get(docs[3].bid).getDocumentCount()); @@ -1047,7 +1041,7 @@ TEST_F("require that removes are not remembered", SearchableFeedViewFixture) TEST_F("require that heartbeat propagates to index- and attributeadapter", SearchableFeedViewFixture) { - f.runInMaster([&] () { f.fv.heartBeat(2); }); + f.runInMasterAndSyncAll([&]() { f.fv.heartBeat(2); }); EXPECT_EQUAL(1, f.miw._heartBeatCount); EXPECT_EQUAL(1, f.maw._heartBeatCount); } @@ -1143,7 +1137,7 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc f.compactLidSpaceAndWait(2); // performIndexForceCommit in index thread, then completion callback // in master thread. - EXPECT_TRUE(assertThreadObserver(7, 6, 6, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(7, 7, 7, f.writeServiceObserver())); EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); @@ -1159,32 +1153,29 @@ TEST_F("require that compactLidSpace() doesn't propagate to " EXPECT_TRUE(assertThreadObserver(5, 4, 4, f.writeServiceObserver())); CompactLidSpaceOperation op(0, 2); op.setSerialNum(0); - f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); }); + f.runInMasterAndSyncAll([&]() { f.fv.handleCompactLidSpace(op); }); // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread - EXPECT_TRUE(assertThreadObserver(6, 5, 4, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(6, 6, 5, f.writeServiceObserver())); EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); } -TEST_F("require that compactLidSpace() propagates to attributeadapter", - FastAccessFeedViewFixture) +TEST_F("require that compactLidSpace() propagates to attributeadapter", FastAccessFeedViewFixture) { f.populateBeforeCompactLidSpace(); f.compactLidSpaceAndWait(2); EXPECT_EQUAL(2u, f.maw._wantedLidLimit); } -TEST_F("require that compactLidSpace() propagates to index writer", - SearchableFeedViewFixture) +TEST_F("require that compactLidSpace() propagates to index writer", SearchableFeedViewFixture) { f.populateBeforeCompactLidSpace(); f.compactLidSpaceAndWait(2); EXPECT_EQUAL(2u, f.miw._wantedLidLimit); } -TEST_F("require that commit is not implicitly called", - SearchableFeedViewFixture) +TEST_F("require that commit is not implicitly called", SearchableFeedViewFixture) { DocumentContext dc = f.doc1(); f.putAndWait(dc); @@ -1204,8 +1195,7 @@ TEST_F("require that commit is not implicitly called", f.forceCommitAndWait(); } -TEST_F("require that forceCommit updates docid limit", - SearchableFeedViewFixture) +TEST_F("require that forceCommit updates docid limit", SearchableFeedViewFixture) { DocumentContext dc = f.doc1(); f.putAndWait(dc); diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index eba87447f86..7ccaa6e9fbf 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> @@ -240,13 +241,24 @@ struct FixtureBase { } template <typename FunctionType> + void runInMasterAndSyncAll(FunctionType func) { + test::runInMasterAndSyncAll(writeService, func); + } + template <typename FunctionType> + void runInMasterAndSync(FunctionType func) { + test::runInMasterAndSync(writeService, func); + } + template <typename FunctionType> void runInMaster(FunctionType func) { test::runInMaster(writeService, func); } void force_commit() { - runInMaster([this] () { static_cast<IFeedView&>(*feedview).forceCommit(serial_num); }); - writeService.sync_all_executors(); + vespalib::Gate gate; + runInMaster([this, &gate]() { + feedview->forceCommit(search::CommitParam(serial_num), std::make_shared<vespalib::GateCallback>(gate)); + }); + gate.await(); } }; @@ -281,7 +293,7 @@ TEST_F("require that prepareMove sets target db document id", Fixture) { Document::SP doc(new Document); MoveOperation op(BucketId(20, 42), Timestamp(10), doc, 1, subdb_id + 1); - f.runInMaster([&] () { f.feedview->prepareMove(op); }); + f.runInMasterAndSync([&]() { f.feedview->prepareMove(op); }); DbDocumentId targetId = op.getDbDocumentId(); EXPECT_EQUAL(subdb_id, targetId.getSubDbId()); @@ -309,8 +321,8 @@ TEST_F("require that handleMove() adds document to target and removes it from so { // move from (subdb_id + 1) -> this (subdb_id) MoveOperation::UP op = makeMoveOp(DbDocumentId(subdb_id + 1, 1), subdb_id); TEST_DO(f.assertPutCount(0)); - f.runInMaster([&]() { f.feedview->prepareMove(*op); }); - f.runInMaster([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); }); + f.runInMasterAndSync([&]() { f.feedview->prepareMove(*op); }); + f.runInMasterAndSyncAll([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); }); TEST_DO(f.assertPutCount(1)); TEST_DO(f.assertAndClearMoveOp()); lid = op->getDbDocumentId().getLid(); @@ -322,7 +334,7 @@ TEST_F("require that handleMove() adds document to target and removes it from so MoveOperation::UP op = makeMoveOp(DbDocumentId(subdb_id, 1), subdb_id + 1); op->setDbDocumentId(DbDocumentId(subdb_id + 1, 1)); TEST_DO(f.assertRemoveCount(0)); - f.runInMaster([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); }); + f.runInMasterAndSyncAll([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); }); EXPECT_FALSE(f.metaStore->validLid(lid)); TEST_DO(f.assertRemoveCount(1)); TEST_DO(f.assertAndClearMoveOp()); @@ -334,19 +346,23 @@ TEST_F("require that handleMove() handles move within same subdb and propagates Document::SP doc(new Document); DocumentId doc1id("id:test:foo:g=foo:1"); uint32_t docSize = 1; - f.runInMaster([&] () { f.metaStore->put(doc1id.getGlobalId(), - doc1id.getGlobalId().convertToBucketId(), - Timestamp(9), docSize, 1, 0u); }); - f.runInMaster([&] () { f.metaStore->put(doc->getId().getGlobalId(), - doc->getId().getGlobalId().convertToBucketId(), - Timestamp(10), docSize, 2, 0u); }); - f.runInMaster([&] () { f.metaStore->remove(1, 0u); }); + f.runInMasterAndSync([&]() { + f.metaStore->put(doc1id.getGlobalId(), + doc1id.getGlobalId().convertToBucketId(), + Timestamp(9), docSize, 1, 0u); + }); + f.runInMasterAndSync([&]() { + f.metaStore->put(doc->getId().getGlobalId(), + doc->getId().getGlobalId().convertToBucketId(), + Timestamp(10), docSize, 2, 0u); + }); + f.runInMasterAndSync([&]() { f.metaStore->remove(1, 0u); }); f.metaStore->removes_complete({ 1 }); MoveOperation::UP op = makeMoveOp(doc, DbDocumentId(subdb_id, 2), subdb_id); op->setTargetLid(1); TEST_DO(f.assertPutCount(0)); TEST_DO(f.assertRemoveCount(0)); - f.runInMaster([&] () { f.feedview->handleMove(*op, f.beginMoveOp()); }); + f.runInMasterAndSyncAll([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); }); TEST_DO(f.assertPutCount(1)); TEST_DO(f.assertRemoveCount(1)); TEST_DO(f.assertAndClearMoveOp()); @@ -366,7 +382,7 @@ TEST_F("require that prune removed documents removes documents", PruneRemovedDocumentsOperation op(lids->getDocIdLimit(), subdb_id); op.setLidsToRemove(lids); op.setSerialNum(1); // allows use of meta store. - f.runInMaster([&] () { f.feedview->handlePruneRemovedDocuments(op); }); + f.runInMasterAndSyncAll([&]() { f.feedview->handlePruneRemovedDocuments(op); }); EXPECT_EQUAL(2, f.removeCount); EXPECT_FALSE(f.metaStore->validLid(1)); @@ -382,7 +398,7 @@ TEST_F("require that heartbeat propagates and commits meta store", Fixture) EXPECT_EQUAL(0, f.feedview->heartBeatIndexedFieldsCount); EXPECT_EQUAL(0, f.feedview->heartBeatAttributesCount); EXPECT_EQUAL(0, f.heartbeatCount); - f.runInMaster([&] () { f.feedview->heartBeat(2); }); + f.runInMasterAndSyncAll([&]() { f.feedview->heartBeat(2); }); EXPECT_EQUAL(2u, f.metaStore->getStatus().getLastSyncToken()); EXPECT_EQUAL(1, f.feedview->heartBeatIndexedFieldsCount); EXPECT_EQUAL(1, f.feedview->heartBeatAttributesCount); diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 50a8349b859..a59b3e9bc6f 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -43,7 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con TEST_DO(f.assertIndexingThreads(3, 18)); TEST_DO(f.assertIndexingThreads(4, 19)); TEST_DO(f.assertIndexingThreads(4, 24)); - TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4 + TEST_DO(f.assertIndexingThreads(11, 64)); } TEST_F("require that indexing threads is always >= 1", Fixture(0)) diff --git a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt index 3c9a4e05b50..c3d70bd7410 100644 --- a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt @@ -11,5 +11,4 @@ vespa_add_executable(searchcore_documentmetastore_test_app TEST searchcore_fconfig GTest::GTest ) -vespa_add_test(NAME searchcore_documentmetastore_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/documentmetastore_test.sh - DEPENDS searchcore_documentmetastore_test_app) +vespa_add_test(NAME searchcore_documentmetastore_test_app COMMAND searchcore_documentmetastore_test_app) diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 722ea25a14e..c96b0825789 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -336,20 +336,22 @@ TEST(DocumentMetaStore, gids_can_be_cleared) assertLid(1, gid1, dms); EXPECT_EQ(1u, dms.getNumUsedLids()); EXPECT_TRUE(dms.remove(1, 0u)); - dms.removes_complete({ 1 }); + dms.commit(); EXPECT_EQ(0u, dms.getNumUsedLids()); EXPECT_TRUE(!dms.getGid(1, gid)); EXPECT_TRUE(!dms.getLid(gid1, lid)); + dms.removes_complete({ 1 }); // reuse lid addGid(dms, gid2, bucketId2, time2); assertGid(gid2, 1, dms); assertLid(1, gid2, dms); EXPECT_EQ(1u, dms.getNumUsedLids()); EXPECT_TRUE(dms.remove(1, 0u)); - dms.removes_complete({ 1 }); + dms.commit(); EXPECT_EQ(0u, dms.getNumUsedLids()); EXPECT_TRUE(!dms.getGid(1, gid)); EXPECT_TRUE(!dms.getLid(gid2, lid)); + dms.removes_complete({ 1 }); EXPECT_TRUE(!dms.remove(1, 0u)); // not used EXPECT_TRUE(!dms.remove(2, 0u)); // outside range } @@ -519,6 +521,7 @@ TEST(DocumentMetaStoreTest, gids_can_be_saved_and_loaded) EXPECT_EQ(numLids + 1, dms2.getNumDocs()); EXPECT_EQ(numLids - (3 - i), dms2.getNumUsedLids()); } + vespalib::unlink("documentmetastore2.dat"); } TEST(DocumentMetaStoreTest, bucket_used_bits_are_lbounded_at_load_time) @@ -544,6 +547,7 @@ TEST(DocumentMetaStoreTest, bucket_used_bits_are_lbounded_at_load_time) BucketId expected_bucket(storage::spi::BucketLimits::MinUsedBits, gid.convertToBucketId().getRawId()); assertGid(gid, lid, dms2, expected_bucket, Timestamp(1000)); + vespalib::unlink("documentmetastore2.dat"); } TEST(DocumentMetaStore, stats_are_updated) @@ -1732,6 +1736,7 @@ remove(uint32_t startLid, uint32_t shrinkTarget, DocumentMetaStore &dms) { for (uint32_t lid = startLid; lid >= shrinkTarget; --lid) { dms.remove(lid, 0u); + dms.commit(); dms.removes_complete({ lid }); } } @@ -1765,8 +1770,6 @@ TEST(DocumentMetaStoreTest, shrink_via_flush_target_works) DummyFileHeaderContext fileHeaderContext; DummyTlsSyncer dummyTlsSyncer; HwInfo hwInfo; - vespalib::rmdir("dmsflush", true); - vespalib::mkdir("dmsflush"); using Type = IFlushTarget::Type; using Component = IFlushTarget::Component; IFlushTarget::SP ft(std::make_shared<ShrinkLidSpaceFlushTarget> @@ -1903,6 +1906,8 @@ TEST(DocumentMetaStoreTest, document_sizes_are_saved) assertSize(dms4, 1, 1); assertSize(dms4, 2, 1); assertSize(dms4, 3, 1); + vespalib::unlink("documentmetastore3.dat"); + vespalib::unlink("documentmetastore4.dat"); } namespace { @@ -1944,12 +1949,12 @@ TEST(DocumentMetaStoreTest, multiple_lids_can_be_removed_with_removeBatch) assertLidGidFound(4, dms); dms.removeBatch({1, 3}, 5); - dms.removes_complete({1, 3}); - + dms.commit(); assertLidGidNotFound(1, dms); assertLidGidFound(2, dms); assertLidGidNotFound(3, dms); assertLidGidFound(4, dms); + dms.removes_complete({1, 3}); } class MockOperationListener : public documentmetastore::OperationListener { diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh deleted file mode 100755 index 199a71106f8..00000000000 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -$VALGRIND ./searchcore_documentmetastore_test_app -rm -rf documentmetastore*.dat -rm -rf dmsflush diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 33b58e32669..8265585a038 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -136,8 +136,13 @@ public: } template <typename FunctionType> - void runInMaster(FunctionType func) { - test::runInMaster(_writeService, func); + void runInMasterAndSyncAll(FunctionType func) { + test::runInMasterAndSyncAll(_writeService, func); + } + + template <typename FunctionType> + void runInMasterAndSync(FunctionType func) { + test::runInMasterAndSync(_writeService, func); } void cycledLids(const std::vector<uint32_t> &lids) { @@ -155,19 +160,16 @@ public: } void delayReuse(uint32_t lid) { - runInMaster([&] () { _lidReuseDelayer->delayReuse(lid); } ); + runInMasterAndSync([&]() { _lidReuseDelayer->delayReuse(lid); }); } void delayReuse(const std::vector<uint32_t> &lids) { - runInMaster([&] () { _lidReuseDelayer->delayReuse(lids); }); + runInMasterAndSync([&]() { _lidReuseDelayer->delayReuse(lids); }); } void commit() { - runInMaster([&] () { cycleLids(_lidReuseDelayer->getReuseLids()); }); + runInMasterAndSyncAll([&]() { cycleLids(_lidReuseDelayer->getReuseLids()); }); } - - void sync() { _writeService.sync_all_executors(); } - }; TEST_F("require that nothing happens before free list is active", Fixture) diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index d34e2ae667e..d6bbc77aa09 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -104,8 +104,6 @@ void push_documents_and_wait(search::memoryindex::DocumentInverter &inverter) { gate.await(); } -std::shared_ptr<vespalib::IDestructorCallback> emptyDestructorCallback; - struct IndexManagerTest : public ::testing::Test { SerialNum _serial_num; IndexManagerDummyReconfigurer _reconfigurer; @@ -128,7 +126,6 @@ struct IndexManagerTest : public ::testing::Test { { removeTestData(); vespalib::mkdir(index_dir, false); - _writeService.sync_all_executors(); resetIndexManager(); } @@ -138,22 +135,31 @@ struct IndexManagerTest : public ::testing::Test { template <class FunctionType> inline void runAsMaster(FunctionType &&function) { - _writeService.master().execute(makeLambdaTask(std::move(function))); - _writeService.master().sync(); + vespalib::Gate gate; + _writeService.master().execute(makeLambdaTask([&gate,function = std::move(function)]() { + function(); + gate.countDown(); + })); + gate.await(); } template <class FunctionType> inline void runAsIndex(FunctionType &&function) { - _writeService.index().execute(makeLambdaTask(std::move(function))); - _writeService.index().sync(); + vespalib::Gate gate; + _writeService.index().execute(makeLambdaTask([&gate,function = std::move(function)]() { + function(); + gate.countDown(); + })); + gate.await(); } void flushIndexManager(); Document::UP addDocument(uint32_t docid); void resetIndexManager(); void removeDocument(uint32_t docId, SerialNum serialNum) { vespalib::Gate gate; - runAsIndex([&]() { _index_manager->removeDocument(docId, serialNum); - _index_manager->commit(serialNum, std::make_shared<vespalib::GateCallback>(gate)); - }); + runAsIndex([&]() { + _index_manager->removeDocument(docId, serialNum); + _index_manager->commit(serialNum, std::make_shared<vespalib::GateCallback>(gate)); + }); gate.await(); } void removeDocument(uint32_t docId) { diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index b64e972c4cf..3474a4297c7 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -38,6 +38,7 @@ class ListenerStats { uint32_t _createdListeners; uint32_t _registeredListeners; uint32_t _destroyedListeners; + std::vector<GlobalId> _initial_removes; public: ListenerStats() noexcept @@ -46,7 +47,8 @@ public: _removeChanges(0u), _createdListeners(0u), _registeredListeners(0u), - _destroyedListeners(0u) + _destroyedListeners(0u), + _initial_removes() { } @@ -64,7 +66,7 @@ public: ++_removeChanges; } void markCreatedListener() { lock_guard guard(_lock); ++_createdListeners; } - void markRegisteredListener() { lock_guard guard(_lock); ++_registeredListeners; } + void markRegisteredListener(const std::vector<GlobalId>& removes) { lock_guard guard(_lock); ++_registeredListeners; _initial_removes = removes; } void markDestroyedListener() { lock_guard guard(_lock); ++_destroyedListeners; } uint32_t getCreatedListeners() const { return _createdListeners; } @@ -84,6 +86,7 @@ public: EXPECT_EQUAL(expPutChanges, _putChanges); EXPECT_EQUAL(expRemoveChanges, _removeChanges); } + const std::vector<GlobalId>& get_initial_removes() const noexcept { return _initial_removes; } }; class MyListener : public IGidToLidChangeListener @@ -105,7 +108,7 @@ public: ~MyListener() override { _stats.markDestroyedListener(); } void notifyPutDone(IDestructorCallbackSP, GlobalId, uint32_t) override { _stats.notifyPutDone(); } void notifyRemove(IDestructorCallbackSP, GlobalId) override { _stats.notifyRemove(); } - void notifyRegistered() override { _stats.markRegisteredListener(); } + void notifyRegistered(const std::vector<GlobalId>& removes) override { _stats.markRegisteredListener(removes); } const vespalib::string &getName() const override { return _name; } const vespalib::string &getDocTypeName() const override { return _docTypeName; } }; @@ -233,6 +236,16 @@ TEST_F("Test that we keep old listener when registering duplicate", Fixture) TEST_DO(stats.assertListeners(2, 1, 1)); } +TEST_F("Test that pending removes are passed on to new listener", Fixture) +{ + auto& stats = f.addStats(); + auto listener = std::make_unique<MyListener>(stats, "test1", "testdoc"); + f.notifyRemove(toGid(doc1), 20); + f.addListener(std::move(listener)); + EXPECT_TRUE((std::vector<GlobalId>{ toGid(doc1) }) == stats.get_initial_removes()); + f.commit(); +} + class StatsFixture : public Fixture { ListenerStats &_stats; diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index 15281563b93..e3b1fd3aa15 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -103,8 +103,8 @@ struct Fixture gate.await(); } - void notifyListenerRegistered() { - _listener->notifyRegistered(); + void notifyListenerRegistered(const std::vector<GlobalId>& removes) { + _listener->notifyRegistered(removes); } }; @@ -144,7 +144,7 @@ TEST_F("Test that target lids are populated when listener is registered", Fixtur std::make_shared<MyGidToLidMapperFactory>(); f._attr->setGidToLidMapperFactory(factory); f.allocListener(); - f.notifyListenerRegistered(); + f.notifyListenerRegistered({}); TEST_DO(f.assertTargetLid(10, 1)); TEST_DO(f.assertTargetLid(17, 2)); TEST_DO(f.assertTargetLid(10, 3)); @@ -152,6 +152,31 @@ TEST_F("Test that target lids are populated when listener is registered", Fixtur TEST_DO(f.assertNoTargetLid(5)); } +TEST_F("Test that removed target lids are pruned when listener is registered", Fixture) +{ + f.ensureDocIdLimit(6); + f.set(1, toGid(doc1)); + f.set(2, toGid(doc2)); + f.set(3, toGid(doc1)); + f.set(4, toGid(doc3)); + f.commit(); + TEST_DO(f.assertTargetLid(0, 1)); + TEST_DO(f.assertTargetLid(0, 2)); + TEST_DO(f.assertTargetLid(0, 3)); + TEST_DO(f.assertTargetLid(0, 4)); + TEST_DO(f.assertNoTargetLid(5)); + std::shared_ptr<search::IGidToLidMapperFactory> factory = + std::make_shared<MyGidToLidMapperFactory>(); + f._attr->setGidToLidMapperFactory(factory); + f.allocListener(); + f.notifyListenerRegistered({ toGid(doc1) }); + TEST_DO(f.assertTargetLid(0, 1)); + TEST_DO(f.assertTargetLid(17, 2)); + TEST_DO(f.assertTargetLid(0, 3)); + TEST_DO(f.assertTargetLid(0, 4)); + TEST_DO(f.assertNoTargetLid(5)); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp index 8cff0b1ea6c..c2efa8c2389 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp @@ -26,7 +26,7 @@ public: ~MyListener() override { } void notifyPutDone(IDestructorCallbackSP, document::GlobalId, uint32_t) override { } void notifyRemove(IDestructorCallbackSP, document::GlobalId) override { } - void notifyRegistered() override { } + void notifyRegistered(const std::vector<document::GlobalId>&) override { } const vespalib::string &getName() const override { return _name; } const vespalib::string &getDocTypeName() const override { return _docTypeName; } }; diff --git a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp index eccae8fe8ab..d5421eaaeca 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp @@ -134,7 +134,9 @@ SpiBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& docum auto provider = get_provider(bucket); if (provider) { Bucket spi_bucket(bucket); - provider->removeAsync(spi_bucket, Timestamp(timestamp), document_id, context, std::make_unique<MyOperationComplete>(provider, _errors, spi_bucket, tracker)); + std::vector<storage::spi::PersistenceProvider::TimeStampAndDocumentId> ids; + ids.emplace_back(Timestamp(timestamp), document_id); + provider->removeAsync(spi_bucket, std::move(ids), context, std::make_unique<MyOperationComplete>(provider, _errors, spi_bucket, tracker)); } else { ++_errors; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index f3d90d37e42..77b6d8d1392 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -259,7 +259,7 @@ applyCommit(CommitParam param, AttributeWriter::OnWriteDoneType , AttributeVecto void applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum, AttributeVector &attr) { - if (attr.getStatus().getLastSyncToken() < serialNum) { + if (attr.getStatus().getLastSyncToken() <= serialNum) { /* * If the attribute is an empty placeholder attribute due to * later config changes removing the attribute then it might @@ -657,7 +657,6 @@ AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr) _attributeFieldWriter(_mgr->getAttributeFieldWriter()), _shared_executor(_mgr->get_shared_executor()), _writeContexts(), - _dataType(nullptr), _hasStructFieldAttribute(false), _attrMap() { @@ -673,9 +672,18 @@ void AttributeWriter::setupAttributeMapping() { } -AttributeWriter::~AttributeWriter() -{ - _attributeFieldWriter.sync_all(); +AttributeWriter::~AttributeWriter() { + vespalib::Gate gate; + drain(std::make_shared<vespalib::GateCallback>(gate)); + gate.await(); +} + +void +AttributeWriter::drain(OnWriteDoneType onDone) { + + for (const auto &wc : _writeContexts) { + _attributeFieldWriter.executeLambda(wc.getExecutorId(), [onDone] () { (void) onDone; }); + } } std::vector<search::AttributeVector *> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 726005ae04b..dc543c19222 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -83,13 +83,11 @@ public: private: using AttrMap = vespalib::hash_map<vespalib::string, AttributeWithInfo>; std::vector<WriteContext> _writeContexts; - const DataType *_dataType; bool _hasStructFieldAttribute; AttrMap _attrMap; void setupWriteContexts(); void setupAttributeMapping(); - void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool allAttributes, OnWriteDoneType onWriteDone); void internalRemove(SerialNum serialNum, DocumentIdT lid, OnWriteDoneType onWriteDone); @@ -120,6 +118,7 @@ public: void onReplayDone(uint32_t docIdLimit) override; bool hasStructFieldAttribute() const override; + void drain(OnWriteDoneType onWriteDone) override; // Should only be used for unit testing. const std::vector<WriteContext>& get_write_contexts() const { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 032be9e1dc8..eee6264b9f4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -22,6 +22,8 @@ #include <vespa/vespalib/util/threadexecutor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/gate.h> +#include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attributemanager"); @@ -59,14 +61,14 @@ search::SerialNum estimateShrinkSerialNum(const AttributeVector &attr) return std::max(attr.getStatus().getLastSyncToken(), serialNum); } -std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector::SP &attr, vespalib::ISequencedTaskExecutor &attributeFieldWriter, AttributeDiskLayout &diskLayout) +std::shared_ptr<ShrinkLidSpaceFlushTarget> +allocShrinker(const AttributeVector::SP &attr, vespalib::ISequencedTaskExecutor & executor, AttributeDiskLayout &diskLayout) { using Type = IFlushTarget::Type; using Component = IFlushTarget::Component; - auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, - attributeFieldWriter.getExecutorIdFromName( - attr->getNamePrefix())); + auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, executor, + executor.getExecutorIdFromName(attr->getNamePrefix())); const vespalib::string &name = attr->getName(); auto dir = diskLayout.createAttributeDir(name); search::SerialNum shrinkSerialNum = estimateShrinkSerialNum(*attr); @@ -167,24 +169,32 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, const Spec &newSpec, Spec::AttributeList &toBeAdded) { - for (const auto &aspec : newSpec.getAttributes()) { - AttributeVector::SP av = currMgr.findAttribute(aspec.getName()); - if (matchingTypes(av, aspec.getConfig())) { // transfer attribute - LOG(debug, "Transferring attribute vector '%s' with %u docs and serial number %" PRIu64 " from current manager", - av->getName().c_str(), av->getNumDocs(), av->getStatus().getLastSyncToken()); - auto wrap = currMgr.findFlushable(aspec.getName()); - assert(wrap != nullptr); - auto shrinker = wrap->getShrinker(); - assert(shrinker); - addAttribute(AttributeWrap::normalAttribute(av), shrinker); - auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix()); - auto cfg = aspec.getConfig(); - _attributeFieldWriter.execute(id, [av, cfg]() { av->update_config(cfg); }); - } else { - toBeAdded.push_back(aspec); + vespalib::Gate gate; + { + auto gateCallback = std::make_shared<vespalib::GateCallback>(gate); + for (const auto &aspec: newSpec.getAttributes()) { + AttributeVector::SP av = currMgr.findAttribute(aspec.getName()); + if (matchingTypes(av, aspec.getConfig())) { // transfer attribute + LOG(debug, + "Transferring attribute vector '%s' with %u docs and serial number %" PRIu64 " from current manager", + av->getName().c_str(), av->getNumDocs(), av->getStatus().getLastSyncToken()); + auto wrap = currMgr.findFlushable(aspec.getName()); + assert(wrap != nullptr); + auto shrinker = wrap->getShrinker(); + assert(shrinker); + addAttribute(AttributeWrap::normalAttribute(av), shrinker); + auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix()); + auto cfg = aspec.getConfig(); + _attributeFieldWriter.execute(id, [av, cfg, gateCallback]() { + (void) gateCallback; + av->update_config(cfg); + }); + } else { + toBeAdded.push_back(aspec); + } } } - _attributeFieldWriter.sync_all(); + gate.await(); } void @@ -585,7 +595,7 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> } void -AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const +AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const { for (const auto &attr : _attributes) { if (attr.second.isExtra()) { @@ -594,7 +604,10 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) } AttributeVector::SP attrsp = attr.second.getAttribute(); _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()), - [attrsp, func]() { (*func)(*attrsp); }); + [attrsp, func, onDone]() { + (void) onDone; + (*func)(*attrsp); + }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index e2b9550435d..08e2d511d70 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -178,7 +178,7 @@ public: const std::vector<search::AttributeVector *> &getWritableAttributes() const override; void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const override; - void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override; + void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const override; void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index c7ab83ae590..5f162281d96 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -206,7 +206,7 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFun } void -FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const +FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const { // Run by document db master thread std::vector<AttributeGuard> completeList; @@ -217,7 +217,10 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> // Name must be extracted in document db master thread or attribute // writer thread attributeFieldWriter.execute(attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()), - [attrsp, func]() { (*func)(*attrsp); }); + [attrsp, func, onDone]() { + (void) onDone; + (*func)(*attrsp); + }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 1ae5f452218..1512ab32d62 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -52,7 +52,7 @@ public: search::AttributeVector * getWritableAttribute(const vespalib::string &name) const override; const std::vector<search::AttributeVector *> & getWritableAttributes() const override; void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const override; - void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override; + void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const override; ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index d55cd45d014..b8968ba9d2e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -31,7 +31,7 @@ class ImportedAttributesRepo; struct IAttributeManager : public search::IAttributeManager { using SP = std::shared_ptr<IAttributeManager>; - using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; + using OnDone = std::shared_ptr<vespalib::IDestructorCallback>; using IAttributeFunctor = search::attribute::IAttributeFunctor; using IConstAttributeFunctor = search::attribute::IConstAttributeFunctor; @@ -98,7 +98,7 @@ struct IAttributeManager : public search::IAttributeManager virtual const std::vector<search::AttributeVector *> &getWritableAttributes() const = 0; virtual void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const = 0; - virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const = 0; + virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const = 0; virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h index 1e959a4fb77..0f739f6ffea 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h @@ -64,6 +64,7 @@ public: virtual void onReplayDone(uint32_t docIdLimit) = 0; virtual bool hasStructFieldAttribute() const = 0; + virtual void drain(OnWriteDoneType onWriteDone) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index 8fb1415cee8..1342324b727 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -28,7 +28,6 @@ DocumentStoreAdapter::writeStringField(const char * buf, uint32_t buflen, ResTyp case RES_STRING: return _resultPacker.AddString(buf, buflen); case RES_LONG_STRING: - case RES_XMLSTRING: case RES_JSONSTRING: return _resultPacker.AddLongString(buf, buflen); default: @@ -57,7 +56,6 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) return _resultPacker.AddDouble(value.getAsDouble()); case RES_STRING: case RES_LONG_STRING: - case RES_XMLSTRING: case RES_JSONSTRING: { if (value.getClass().inherits(LiteralFieldValueB::classId)) { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 778273c9777..3170654409b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -207,6 +207,8 @@ void DocumentMetaStore::onCommit() { if (consider_compact_gid_to_lid_map()) { + incGeneration(); + _changesSinceCommit = 0; _gidToLidMap.compact_worst(); _gid_to_lid_map_write_itr_prepare_serial_num = 0u; _gid_to_lid_map_write_itr.begin(_gidToLidMap.getRoot()); @@ -690,7 +692,7 @@ DocumentMetaStore::removeBatch(const std::vector<DocId> &lidsToRemove, const uin bucketdb::Guard bucketGuard = _bucketDB->takeGuard(); bucketGuard->remove_batch(bdb_removed, _subDbType); } - incGeneration(); + ++_changesSinceCommit; if (_op_listener) { _op_listener->notify_remove_batch(); } diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h index 758281794b0..bc0d09e5cd4 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h +++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h @@ -41,7 +41,6 @@ class TaskRunner { } void setDone() { execute(std::move(_doneTask)); } const InitializerTask::SP &rootTask() { return _rootTask; } - void schedulePoll(); }; void getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks, TaskSet &checked); void setTaskRunning(InitializerTask &task); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 9d9e1e0a344..7e21e619419 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -368,23 +368,61 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do } void -PersistenceEngine::removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context&, OperationComplete::UP onComplete) +PersistenceEngine::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP onComplete) +{ + if (ids.size() == 1) { + removeAsyncSingle(b, ids[0].first, ids[0].second, std::move(onComplete)); + } else { + removeAsyncMulti(b, std::move(ids), std::move(onComplete)); + } +} + +void +PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) { + ReadGuard rguard(_rwMutex); + //TODO Group per document type/handler and handle in one go. + for (const TimeStampAndDocumentId & stampedId : ids) { + const document::DocumentId & id = stampedId.second; + if (!id.hasDocType()) { + return onComplete->onComplete( + std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, + fmt("Old id scheme not supported in elastic mode (%s)", id.toString().c_str()))); + } + DocTypeName docType(id.getDocType()); + IPersistenceHandler *handler = getHandler(rguard, b.getBucketSpace(), docType); + if (!handler) { + return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, + fmt("No handler for document type '%s'", + docType.toString().c_str()))); + } + } + auto transportContext = std::make_shared<AsyncRemoveTransportContext>(ids.size(), std::move(onComplete)); + for (const TimeStampAndDocumentId & stampedId : ids) { + const document::DocumentId & id = stampedId.second; + DocTypeName docType(id.getDocType()); + IPersistenceHandler *handler = getHandler(rguard, b.getBucketSpace(), docType); + handler->handleRemove(feedtoken::make(transportContext), b, stampedId.first, id); + } +} + +void +PersistenceEngine::removeAsyncSingle(const Bucket& b, Timestamp t, const DocumentId& id, OperationComplete::UP onComplete) { ReadGuard rguard(_rwMutex); LOG(spam, "remove(%s, %" PRIu64 ", \"%s\")", b.toString().c_str(), - static_cast<uint64_t>(t.getValue()), did.toString().c_str()); - if (!did.hasDocType()) { + static_cast<uint64_t>(t.getValue()), id.toString().c_str()); + if (!id.hasDocType()) { return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, - fmt("Old id scheme not supported in elastic mode (%s)", did.toString().c_str()))); + fmt("Old id scheme not supported in elastic mode (%s)", id.toString().c_str()))); } - DocTypeName docType(did.getDocType()); + DocTypeName docType(id.getDocType()); IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR, fmt("No handler for document type '%s'", docType.toString().c_str()))); } auto transportContext = std::make_shared<AsyncTransportContext>(1, std::move(onComplete)); - handler->handleRemove(feedtoken::make(std::move(transportContext)), b, t, did); + handler->handleRemove(feedtoken::make(std::move(transportContext)), b, t, id); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index fe564d01459..7c8040fae9d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -88,7 +88,8 @@ private: void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; std::shared_ptr<BucketExecutor> get_bucket_executor() noexcept { return _bucket_executor.lock(); } - + void removeAsyncSingle(const Bucket&, Timestamp, const document::DocumentId &id, OperationComplete::UP); + void removeAsyncMulti(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP); public: typedef std::unique_ptr<PersistenceEngine> UP; @@ -106,7 +107,7 @@ public: void setActiveStateAsync(const Bucket&, BucketInfo::ActiveState, OperationComplete::UP) override; BucketInfoResult getBucketInfo(const Bucket&) const override; void putAsync(const Bucket &, Timestamp, storage::spi::DocumentSP, Context &context, OperationComplete::UP) override; - void removeAsync(const Bucket&, Timestamp, const document::DocumentId&, Context&, OperationComplete::UP) override; + void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, storage::spi::DocumentUpdateSP, Context&, OperationComplete::UP) override; GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override; CreateIteratorResult diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp index 8a0955b3147..bf70304c4f4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp @@ -5,6 +5,7 @@ using vespalib::make_string; using storage::spi::Result; +using storage::spi::RemoveResult; namespace proton { @@ -30,14 +31,19 @@ TransportMerger::mergeResult(ResultUP result, bool documentWasFound) { } } +Result::UP +TransportMerger::merge(ResultUP accum, ResultUP incoming, bool documentWasFound) { + return documentWasFound ? std::move(incoming) : std::move(accum); +} + void TransportMerger::mergeWithLock(ResultUP result, bool documentWasFound) { if (!_result) { _result = std::move(result); } else if (result->hasError()) { _result = std::make_unique<Result>(mergeErrorResults(*_result, *result)); - } else if (documentWasFound) { - _result = std::move(result); + } else { + _result = merge(std::move(_result), std::move(result), documentWasFound); } completeIfDone(); } @@ -93,4 +99,11 @@ AsyncTransportContext::send(ResultUP result, bool documentWasFound) mergeResult(std::move(result), documentWasFound); } +Result::UP +AsyncRemoveTransportContext::merge(ResultUP accum, ResultUP incoming, bool) { + // TODO This can be static cast if necessary. + dynamic_cast<RemoveResult *>(accum.get())->inc_num_removed(dynamic_cast<RemoveResult *>(incoming.get())->num_removed()); + return accum; +} + } // proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h index 91cadfedcd5..b792124136d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h @@ -23,6 +23,7 @@ protected: ~TransportMerger() override; void mergeResult(ResultUP result, bool documentWasFound); virtual void completeIfDone() { } // Called with lock held if necessary on every merge + virtual ResultUP merge(ResultUP accum, ResultUP incoming, bool documentWasFound); ResultUP _result; private: @@ -47,15 +48,10 @@ public: void await() { _latch.await(); } - const UpdateResult &getUpdateResult() const { - return dynamic_cast<const UpdateResult &>(*_result); - } + const Result &getResult() const { return *_result; } - const RemoveResult &getRemoveResult() const { - return dynamic_cast<const RemoveResult &>(*_result); - } }; @@ -77,5 +73,11 @@ public: void send(ResultUP result, bool documentWasFound) override; }; -} // namespace proton +class AsyncRemoveTransportContext : public AsyncTransportContext { +public: + using AsyncTransportContext::AsyncTransportContext; +protected: + ResultUP merge(ResultUP accum, ResultUP incoming, bool documentWasFound) override; +}; +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp index 3de1ac09c09..ed672cc9974 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp @@ -142,7 +142,13 @@ GidToLidChangeHandler::addListener(std::unique_ptr<IGidToLidChangeListener> list } } _listeners.emplace_back(std::move(listener)); - _listeners.back()->notifyRegistered(); + std::vector<GlobalId> removes; + for (auto& change : _pending_changes) { + if (change.is_remove()) { + removes.emplace_back(change.get_gid()); + } + } + _listeners.back()->notifyRegistered(removes); } else { assert(_listeners.empty()); } diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index be03b57e088..7f0f27e3c58 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -1,19 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "gid_to_lid_change_listener.h" +#include <vespa/vespalib/util/gate.h> #include <future> using vespalib::RetainGuard; namespace proton { -GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor &attributeFieldWriter, +GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor & executor, std::shared_ptr<search::attribute::ReferenceAttribute> attr, RetainGuard retainGuard, const vespalib::string &name, const vespalib::string &docTypeName) - : _attributeFieldWriter(attributeFieldWriter), - _executorId(_attributeFieldWriter.getExecutorIdFromName(attr->getNamePrefix())), + : _executor(executor), + _executorId(_executor.getExecutorIdFromName(attr->getNamePrefix())), _attr(std::move(attr)), _retainGuard(std::move(retainGuard)), _name(name), @@ -22,14 +23,16 @@ GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor GidToLidChangeListener::~GidToLidChangeListener() { - _attributeFieldWriter.sync_all(); + vespalib::Gate gate; + _executor.executeLambda(_executorId, [&gate]() { gate.countDown(); }); + gate.await(); } void GidToLidChangeListener::notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) { - _attributeFieldWriter.executeLambda(_executorId, - [this, context=std::move(context), gid, lid]() { + _executor.executeLambda(_executorId, + [this, context=std::move(context), gid, lid]() { (void) context; _attr->notifyReferencedPut(gid, lid); }); @@ -38,21 +41,21 @@ GidToLidChangeListener::notifyPutDone(IDestructorCallbackSP context, document::G void GidToLidChangeListener::notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) { - _attributeFieldWriter.executeLambda(_executorId, - [this, context = std::move(context), gid]() { + _executor.executeLambda(_executorId, + [this, context = std::move(context), gid]() { (void) context; _attr->notifyReferencedRemove(gid); }); } void -GidToLidChangeListener::notifyRegistered() +GidToLidChangeListener::notifyRegistered(const std::vector<document::GlobalId>& removes) { std::promise<void> promise; auto future = promise.get_future(); - _attributeFieldWriter.executeLambda(_executorId, - [this, &promise]() { - _attr->populateTargetLids(); + _executor.executeLambda(_executorId, + [this, &promise, removes(std::move(removes))]() { + _attr->populateTargetLids(removes); promise.set_value(); }); future.wait(); diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h index 28e9684ed86..441c6377128 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h @@ -15,7 +15,7 @@ namespace proton { */ class GidToLidChangeListener : public IGidToLidChangeListener { - vespalib::ISequencedTaskExecutor &_attributeFieldWriter; + vespalib::ISequencedTaskExecutor &_executor; vespalib::ISequencedTaskExecutor::ExecutorId _executorId; std::shared_ptr<search::attribute::ReferenceAttribute> _attr; vespalib::RetainGuard _retainGuard; @@ -31,7 +31,7 @@ public: ~GidToLidChangeListener() override; void notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) override; void notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) override; - void notifyRegistered() override; + void notifyRegistered(const std::vector<document::GlobalId>& removes) override; const vespalib::string &getName() const override; const vespalib::string &getDocTypeName() const override; const std::shared_ptr<search::attribute::ReferenceAttribute> &getReferenceAttribute() const { return _attr; } diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h index 9bdeb2f2e23..75da3adc973 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/stllike/string.h> +#include <vector> namespace document { class GlobalId; } @@ -20,7 +21,7 @@ public: virtual ~IGidToLidChangeListener() { } virtual void notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) = 0; virtual void notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) = 0; - virtual void notifyRegistered() = 0; + virtual void notifyRegistered(const std::vector<document::GlobalId>& removes) = 0; virtual const vespalib::string &getName() const = 0; virtual const vespalib::string &getDocTypeName() const = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 56210fe0c85..511adbe66e9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -54,6 +54,7 @@ vespa_add_library(searchcore_server STATIC health_adapter.cpp heart_beat_job.cpp idocumentdbowner.cpp + ifeedview.cpp ireplayconfig.cpp job_tracked_maintenance_job.cpp lid_space_compaction_handler.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h index 55d0cd479e3..d95dfd1314e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h +++ b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h @@ -13,8 +13,8 @@ namespace proton { struct AttributeWriterFactory : public IAttributeWriterFactory { AttributeWriterFactory() {} - virtual IAttributeWriter::SP create(const IAttributeWriter::SP &old, - const AttributeCollectionSpec &attrSpec) const override + IAttributeWriter::SP create(const IAttributeWriter::SP &old, + const AttributeCollectionSpec &attrSpec) const override { const AttributeWriter &oldAdapter = dynamic_cast<const AttributeWriter &>(*old.get()); const proton::IAttributeManager::SP &oldMgr = oldAdapter.getAttributeManager(); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 427d435aae7..cc05fc7169c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -29,7 +29,6 @@ #include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> #include <vespa/searchcore/proton/reference/document_db_reference_resolver.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_registry.h> -#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/configconverter.h> #include <vespa/searchlib/engine/docsumreply.h> #include <vespa/searchlib/engine/searchreply.h> @@ -104,6 +103,16 @@ public: } }; +template<typename T> +void +forceCommitAndWait(std::shared_ptr<IFeedView> feedView, SerialNum serialNum, T keepAlive) { + vespalib::Gate gate; + using Keep = vespalib::KeepAlive<std::pair<T, std::shared_ptr<IDestructorCallback>>>; + feedView->forceCommit(CommitParam(serialNum), + std::make_shared<Keep>(std::make_pair(std::move(keepAlive), std::make_shared<GateCallback>(gate)))); + gate.await(); +} + } template <typename FunctionType> @@ -196,9 +205,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _transient_usage_provider(std::make_shared<DocumentDBResourceUsageProvider>(*this)), _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, - metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, - DocumentSubDBCollection::Config(protonCfg.numsearcherthreads), - hwInfo), + metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, hwInfo), _maintenanceController(_writeService.master(), sharedExecutor, _refCount, _docTypeName), _jobTrackers(), _calc(), @@ -378,12 +385,8 @@ DocumentDB::enterOnlineState() { // Called by executor thread assert(_writeService.master().isCurrentThread()); - { - vespalib::Gate gate; - // Ensure that all replayed operations are committed to memory structures - _feedView.get()->forceCommit(CommitParam(_feedHandler->getSerialNum()), std::make_shared<GateCallback>(gate)); - gate.await(); - } + // Ensure that all replayed operations are committed to memory structures + _feedView.get()->forceCommitAndWait(CommitParam(_feedHandler->getSerialNum())); (void) _state.enterOnlineState(); // Consider delayed pruning of transaction log and config history @@ -469,11 +472,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum } { bool elidedConfigSave = equalReplayConfig && tlsReplayDone; - vespalib::Gate gate; - // Flush changes to attributes and memory index, cf. visibilityDelay - _feedView.get()->forceCommit(CommitParam(elidedConfigSave ? serialNum : serialNum - 1), - std::make_shared<vespalib::KeepAlive<std::pair<FeedHandler::CommitResult, std::shared_ptr<IDestructorCallback>>>>(std::make_pair(std::move(commit_result), std::make_shared<GateCallback>(gate)))); - gate.await(); + forceCommitAndWait(_feedView.get(), elidedConfigSave ? serialNum : serialNum - 1, std::move(commit_result)); } if (params.shouldMaintenanceControllerChange()) { _maintenanceController.killJobs(); @@ -517,30 +516,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum } } - -void -DocumentDB::performDropFeedView(IFeedView::SP feedView) -{ - // Delays when feed view is dropped. - assert(_writeService.master().isCurrentThread()); - _writeService.attributeFieldWriter().sync_all(); - _writeService.summary().sync(); - - // Feed view is kept alive in the closure's shared ptr. - _writeService.index().execute(makeLambdaTask([this, feedView] () { performDropFeedView2(feedView); })); -} - - -void -DocumentDB::performDropFeedView2(IFeedView::SP feedView) { - // Delays when feed view is dropped. - assert(_writeService.index().isCurrentThread()); - _writeService.indexFieldInverter().sync_all(); - _writeService.indexFieldWriter().sync_all(); - masterExecute([feedView]() { (void) feedView; }); -} - - void DocumentDB::tearDownReferences() { @@ -567,8 +542,10 @@ DocumentDB::close() } // Abort any ongoing maintenance stopMaintenance(); - _writeService.master().sync(); // Complete all tasks that didn't observe shutdown - masterExecute([this]() { tearDownReferences(); }); + masterExecute([this]() { + _feedView.get()->forceCommitAndWait(search::CommitParam(getCurrentSerialNumber())); + tearDownReferences(); + }); _writeService.master().sync(); // Wait until inflight feed operations to this document db has left. // Caller should have removed document DB from feed router. @@ -580,8 +557,11 @@ DocumentDB::close() DocumentDBTaggedMetrics &metrics = getMetrics(); _metricsWireService.cleanAttributes(metrics.ready.attributes); _metricsWireService.cleanAttributes(metrics.notReady.attributes); - _writeService.sync_all_executors(); - masterExecute([this] () { closeSubDBs(); } ); + + masterExecute([this] () { + _feedView.get()->forceCommitAndWait(search::CommitParam(getCurrentSerialNumber())); + closeSubDBs(); + }); _writeService.sync_all_executors(); // What about queued tasks ? _writeService.shutdown(); @@ -912,20 +892,14 @@ void DocumentDB::syncFeedView() { assert(_writeService.master().isCurrentThread()); - IFeedView::SP oldFeedView(_feedView.get()); IFeedView::SP newFeedView(_subDBs.getFeedView()); _maintenanceController.killJobs(); - _writeService.sync_all_executors(); _feedView.set(newFeedView); _feedHandler->setActiveFeedView(newFeedView.get()); _subDBs.createRetrievers(); _subDBs.maintenanceSync(_maintenanceController); - - // Ensure that old feed view is referenced until all index executor tasks - // depending on it has completed. - performDropFeedView(oldFeedView); } bool @@ -990,7 +964,6 @@ void DocumentDB::stopMaintenance() { _maintenanceController.stop(); - _writeService.sync_all_executors(); } void @@ -1023,7 +996,7 @@ DocumentDB::notifyClusterStateChanged(const std::shared_ptr<IBucketStateCalculat if (cfv != nullptr) cfv->setCalculator(newCalc); } - _subDBs.setBucketStateCalculator(newCalc); + _subDBs.setBucketStateCalculator(newCalc, std::shared_ptr<vespalib::IDestructorCallback>()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 6b855cd40a8..391c11df276 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -160,18 +160,6 @@ private: void enterApplyLiveConfigState(); /** - * Drop old field view in a controlled manner. The feed view will - * be kept alive until the index executor is done with all current - * tasks. - * - * Called by executor thread. - * - * @param feedView shared pointer to feed view to be dropped. - */ - void performDropFeedView(IFeedView::SP feedView); - void performDropFeedView2(IFeedView::SP feedView); - - /** * Implements IFeedHandlerOwner */ void onTransactionLogReplayDone() override __attribute__((noinline)); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 4a62a43709f..6576e4ead97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -20,10 +20,6 @@ using vespalib::makeLambdaTask; namespace proton { -DocumentSubDBCollection::Config::Config(size_t numSearchThreads) - : _numSearchThreads(numSearchThreads) -{ } - DocumentSubDBCollection::DocumentSubDBCollection( IDocumentSubDBOwner &owner, search::transactionlog::SyncProxy &tlSyncer, @@ -38,7 +34,6 @@ DocumentSubDBCollection::DocumentSubDBCollection( const vespalib::Clock &clock, std::mutex &configMutex, const vespalib::string &baseDir, - const Config & cfg, const HwInfo &hwInfo) : _subDBs(), _owner(owner), @@ -58,30 +53,22 @@ DocumentSubDBCollection::DocumentSubDBCollection( StoreOnlyDocSubDB::Context context(owner, tlSyncer, getSerialNum, fileHeaderContext, writeService, _bucketDB, *_bucketDBHandler, metrics, configMutex, hwInfo); _subDBs.push_back - (new SearchableDocSubDB( - SearchableDocSubDB::Config( - FastAccessDocSubDB::Config( - StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir, - _readySubDbId, SubDbType::READY), - true, true, false), - cfg.getNumSearchThreads()), - SearchableDocSubDB::Context( - FastAccessDocSubDB::Context(context, metrics.ready.attributes, metricsWireService), - queryLimiter, clock, warmupExecutor))); + (new SearchableDocSubDB(FastAccessDocSubDB::Config( + StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir,_readySubDbId, SubDbType::READY), + true, true, false), + SearchableDocSubDB::Context( + FastAccessDocSubDB::Context(context, metrics.ready.attributes, metricsWireService), + queryLimiter, clock, warmupExecutor))); _subDBs.push_back - (new StoreOnlyDocSubDB( - StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, - _remSubDbId, SubDbType::REMOVED), - context)); + (new StoreOnlyDocSubDB(StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, _remSubDbId, SubDbType::REMOVED), + context)); _subDBs.push_back - (new FastAccessDocSubDB( - FastAccessDocSubDB::Config( - StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir, - _notReadySubDbId, SubDbType::NOTREADY), - true, true, true), - FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService))); + (new FastAccessDocSubDB(FastAccessDocSubDB::Config( + StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir,_notReadySubDbId, SubDbType::NOTREADY), + true, true, true), + FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService))); } @@ -316,11 +303,11 @@ DocumentSubDBCollection::close() } void -DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc) +DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc, OnDone onDone) { _calc = calc; for (auto subDb : _subDBs) { - subDb->setBucketStateCalculator(calc); + subDb->setBucketStateCalculator(calc, onDone); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 880fb57840b..515a886969c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/vespalib/util/idestructorcallback.h> #include <mutex> namespace vespalib { @@ -58,13 +59,7 @@ public: using SubDBVector = std::vector<IDocumentSubDB *>; using const_iterator = SubDBVector::const_iterator; using SerialNum = search::SerialNum; - class Config { - public: - Config(size_t numSearchThreads); - size_t getNumSearchThreads() const noexcept { return _numSearchThreads; } - private: - const size_t _numSearchThreads; - }; + using OnDone = std::shared_ptr<vespalib::IDestructorCallback>; private: using IFeedViewSP = std::shared_ptr<IFeedView>; @@ -99,11 +94,10 @@ public: const vespalib::Clock &clock, std::mutex &configMutex, const vespalib::string &baseDir, - const Config & cfg, const HwInfo &hwInfo); ~DocumentSubDBCollection(); - void setBucketStateCalculator(const IBucketStateCalculatorSP &calc); + void setBucketStateCalculator(const IBucketStateCalculatorSP &calc, OnDone onDone); void createRetrievers(); void maintenanceSync(MaintenanceController &mc); diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 7e0a1851bf5..dcd29c9ddcb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -48,6 +48,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha : _sharedExecutor(sharedExecutor), _masterExecutor(1, stackSize, master_executor), + _shared_field_writer(cfg.shared_field_writer()), _master_task_limit(cfg.master_task_limit()), _indexExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), index_executor)), _summaryExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), summary_executor)), @@ -62,7 +63,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha _index_field_writer_ptr(), _attribute_field_writer_ptr() { - if (cfg.shared_field_writer() == SharedFieldWriterExecutor::INDEX) { + if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) { _field_writer = SequencedTaskExecutor::create(field_writer_executor, cfg.indexingThreads() * 2, cfg.defaultTaskLimit()); _attributeFieldWriter = SequencedTaskExecutor::create(attribute_field_writer_executor, cfg.indexingThreads(), cfg.defaultTaskLimit(), cfg.optimize(), cfg.kindOfwatermark(), cfg.reactionTime()); @@ -70,7 +71,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha _index_field_writer_ptr = _field_writer.get(); _attribute_field_writer_ptr = _attributeFieldWriter.get(); - } else if (cfg.shared_field_writer() == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { + } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { _field_writer = SequencedTaskExecutor::create(field_writer_executor, cfg.indexingThreads() * 3, cfg.defaultTaskLimit(), cfg.optimize(), cfg.kindOfwatermark(), cfg.reactionTime()); _index_field_inverter_ptr = _field_writer.get(); @@ -154,13 +155,28 @@ ExecutorThreadingService::set_task_limits(uint32_t master_task_limit, ExecutorThreadingServiceStats ExecutorThreadingService::getStats() { - return ExecutorThreadingServiceStats(_masterExecutor.getStats(), - _indexExecutor->getStats(), - _summaryExecutor->getStats(), - _sharedExecutor.getStats(), - _index_field_inverter_ptr->getStats(), - _index_field_writer_ptr->getStats(), - _attribute_field_writer_ptr->getStats()); + auto master_stats = _masterExecutor.getStats(); + auto index_stats = _indexExecutor->getStats(); + auto summary_stats = _summaryExecutor->getStats(); + auto shared_stats = _sharedExecutor.getStats(); + if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) { + auto field_writer_stats = _field_writer->getStats(); + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats, + field_writer_stats, + field_writer_stats, + _attribute_field_writer_ptr->getStats()); + } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { + auto field_writer_stats = _field_writer->getStats(); + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats, + field_writer_stats, + field_writer_stats, + field_writer_stats); + } else { + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats, + _index_field_inverter_ptr->getStats(), + _index_field_writer_ptr->getStats(), + _attribute_field_writer_ptr->getStats()); + } } vespalib::ISequencedTaskExecutor & diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 1890ca300e2..629c5043ed7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -2,13 +2,13 @@ #pragma once #include "executor_thread_service.h" +#include "threading_service_config.h" #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/threadstackexecutor.h> namespace proton { class ExecutorThreadingServiceStats; -class ThreadingServiceConfig; /** * Implementation of IThreadingService using 2 underlying thread stack executors @@ -19,6 +19,7 @@ class ExecutorThreadingService : public searchcorespi::index::IThreadingService private: vespalib::ThreadExecutor & _sharedExecutor; vespalib::ThreadStackExecutor _masterExecutor; + ThreadingServiceConfig::SharedFieldWriterExecutor _shared_field_writer; std::atomic<uint32_t> _master_task_limit; std::unique_ptr<vespalib::SyncableThreadExecutor> _indexExecutor; std::unique_ptr<vespalib::SyncableThreadExecutor> _summaryExecutor; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 40a1a1a45f3..f8a6253ead1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -276,7 +276,10 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const } _iFeedView.set(_fastAccessFeedView.get()); if (isNodeRetired()) { - reconfigureAttributesConsideringNodeState(); + // TODO Should probably ahve a similar OnDone callback to applyConfig too. + vespalib::Gate gate; + reconfigureAttributesConsideringNodeState(std::make_shared<vespalib::GateCallback>(gate)); + gate.await(); } } return tasks; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index db2bb7ed2cb..e2b3887c60c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -6,7 +6,6 @@ #include "removedonecontext.h" #include "putdonecontext.h" #include <vespa/searchcore/proton/feedoperation/operations.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> using document::Document; using document::DocumentUpdate; @@ -73,7 +72,7 @@ void FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) { // Drain pending PutDoneContext and ForceCommitContext objects - _writeService.sync_all_executors(); + forceCommitAndWait(search::CommitParam(op.getSerialNum())); _docIdLimit.set(op.getLidLimit()); getAttributeWriter()->compactLidSpace(op.getLidLimit(), op.getSerialNum()); Parent::handleCompactLidSpace(op); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index ea63d59c830..bb03f48882f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -303,7 +303,7 @@ void FeedHandler::performEof() { assert(_writeService.master().isCurrentThread()); - _writeService.sync_all_executors(); + _activeFeedView->forceCommitAndWait(CommitParam(_serialNum)); LOG(debug, "Visiting done for transaction log domain '%s', eof received", _tlsMgr.getDomainName().c_str()); // Replay must be complete if (_replay_end_serial_num != _serialNum) { diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 00849d6ad31..f84352a4558 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -6,6 +6,7 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/searchlib/util/searchable_stats.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/idestructorcallback.h> namespace search::index { class Schema; } @@ -60,6 +61,7 @@ public: using SchemaSP = std::shared_ptr<Schema>; using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>; using IndexConfig = index::IndexConfig; + using OnDone = std::shared_ptr<vespalib::IDestructorCallback>; public: IDocumentSubDB() { } virtual ~IDocumentSubDB() { } @@ -77,7 +79,7 @@ public: virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) = 0; - virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) = 0; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone) = 0; virtual std::shared_ptr<ISearchHandler> getSearchView() const = 0; virtual std::shared_ptr<IFeedView> getFeedView() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp new file mode 100644 index 00000000000..2e07c839d76 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ifeedview.h" +#include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/vespalib/util/gate.h> + +namespace proton { + +void +IFeedView::forceCommitAndWait(CommitParam param) { + vespalib::Gate gate; + forceCommit(param, std::make_shared<vespalib::GateCallback>(gate)); + gate.await(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h index 4feb7d013ea..14e6d6811e7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h @@ -62,6 +62,7 @@ public: virtual void forceCommit(const CommitParam & param, DoneCallback onDone) = 0; void forceCommit(CommitParam param) { forceCommit(param, DoneCallback()); } void forceCommit(search::SerialNum serialNum) { forceCommit(CommitParam(serialNum)); } + void forceCommitAndWait(CommitParam param); virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation & pruneOp) = 0; virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index c4826bba8ea..6b606298026 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -31,9 +31,9 @@ public: }; bool -isRunningOrRunnable(const MaintenanceJobRunner & job, const Executor * master) { +isRunnable(const MaintenanceJobRunner & job, const Executor * master) { return (&job.getExecutor() == master) - ? job.isRunning() + ? false : job.isRunnable(); } @@ -99,14 +99,14 @@ MaintenanceController::killJobs() job->stop(); // Make sure no more tasks are added to the executor } for (auto &job : _jobs) { - while (isRunningOrRunnable(*job, &_masterThread)) { + while (isRunnable(*job, &_masterThread)) { std::this_thread::sleep_for(1ms); } } - JobList tmpJobs = _jobs; + JobList tmpJobs; { Guard guard(_jobsLock); - _jobs.clear(); + tmpJobs.swap(_jobs); } // Hold jobs until existing tasks have been drained _masterThread.execute(makeLambdaTask([this, jobs=std::move(tmpJobs)]() { diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp index 16d74479ebe..9eb0596ff1f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp @@ -77,13 +77,6 @@ MaintenanceJobRunner::MaintenanceJobRunner(Executor &executor, IMaintenanceJob:: } bool -MaintenanceJobRunner::isRunning() const -{ - Guard guard(_lock); - return _running; -} - -bool MaintenanceJobRunner::isRunnable() const { Guard guard(_lock); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h index 151cc302cd3..17f244e6621 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h @@ -31,7 +31,6 @@ public: MaintenanceJobRunner(vespalib::Executor &executor, IMaintenanceJob::UP job); void run() override; void stop(); - bool isRunning() const; bool isRunnable() const; const vespalib::Executor & getExecutor() const { return _executor; } const IMaintenanceJob &getJob() const { return *_job; } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index bf3589457f9..6e87f33e1c6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -29,7 +29,7 @@ using namespace searchcorespi; namespace proton { SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) - : FastAccessDocSubDB(cfg._fastUpdCfg, ctx._fastUpdCtx), + : FastAccessDocSubDB(cfg, ctx._fastUpdCtx), IIndexManager::Reconfigurer(), _indexMgr(), _indexWriter(), @@ -180,9 +180,9 @@ SearchableDocSubDB::propagateFlushConfig() } void -SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) +SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone onDone) { - FastAccessDocSubDB::setBucketStateCalculator(calc); + FastAccessDocSubDB::setBucketStateCalculator(calc, std::move(onDone)); propagateFlushConfig(); } @@ -244,7 +244,7 @@ SearchableDocSubDB::reconfigure(std::unique_ptr<Configure> configure) { assert(_writeService.master().isCurrentThread()); - _writeService.sync_all_executors(); + getFeedView()->forceCommitAndWait(search::CommitParam(_getSerialNum.getSerialNum())); // Everything should be quiet now. diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 2e7aac0a8d3..ea758443cd4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -6,6 +6,7 @@ #include "searchable_feed_view.h" #include "searchview.h" #include "summaryadapter.h" +#include "igetserialnum.h" #include <vespa/eval/eval/value_cache/constant_tensor_loader.h> #include <vespa/eval/eval/value_cache/constant_value_cache.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> @@ -39,15 +40,6 @@ SearchableDocSubDB : public FastAccessDocSubDB, { public: - struct Config { - const FastAccessDocSubDB::Config _fastUpdCfg; - const size_t _numSearcherThreads; - - Config(const FastAccessDocSubDB::Config &fastUpdCfg, size_t numSearcherThreads) - : _fastUpdCfg(fastUpdCfg), - _numSearcherThreads(numSearcherThreads) - { } - }; struct Context { const FastAccessDocSubDB::Context _fastUpdCtx; @@ -113,7 +105,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; - void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override; + void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone onDone) override; void clearViews() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 97e55c37aff..06d174497b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -460,7 +460,7 @@ StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, } void -StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc) { +StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc, OnDone onDone) { bool wasNodeRetired = isNodeRetired(); _nodeRetired = calc->nodeRetired(); if (wasNodeRetired != isNodeRetired()) { @@ -468,16 +468,16 @@ StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCa auto cfg = _dms->getConfig(); cfg.setCompactionStrategy(compactionStrategy); _dms->update_config(cfg); - reconfigureAttributesConsideringNodeState(); + reconfigureAttributesConsideringNodeState(std::move(onDone)); } } void -StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState() { +StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState(OnDone onDone) { search::CompactionStrategy compactionStrategy = computeCompactionStrategy(_lastConfiguredCompactionStrategy); auto attrMan = getAttributeManager(); if (attrMan) { - attrMan->asyncForEachAttribute(std::make_shared<UpdateConfig>(compactionStrategy)); + attrMan->asyncForEachAttribute(std::make_shared<UpdateConfig>(compactionStrategy), std::move(onDone)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 7051722f605..b53dfe89f59 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -146,10 +146,10 @@ protected: vespalib::VarHolder<IFeedView::SP> _iFeedView; std::mutex &_configMutex; HwInfo _hwInfo; + const IGetSerialNum &_getSerialNum; private: - const IGetSerialNum &_getSerialNum; - TlsSyncer _tlsSyncer; - DocumentMetaStoreFlushTarget::SP _dmsFlushTarget; + TlsSyncer _tlsSyncer; + DocumentMetaStoreFlushTarget::SP _dmsFlushTarget; std::shared_ptr<ShrinkLidSpaceFlushTarget> _dmsShrinkTarget; std::shared_ptr<PendingLidTrackerBase> _pendingLidsForCommit; bool _nodeRetired; @@ -183,7 +183,7 @@ protected: StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams(); vespalib::string getSubDbName() const; void reconfigure(const search::LogDocumentStore::Config & protonConfig, const AllocStrategy& alloc_strategy); - void reconfigureAttributesConsideringNodeState(); + void reconfigureAttributesConsideringNodeState(OnDone onDone); public: StoreOnlyDocSubDB(const Config &cfg, const Context &ctx); ~StoreOnlyDocSubDB() override; @@ -203,7 +203,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; - void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override; + void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone onDone) override; ISearchHandler::SP getSearchView() const override { return _iSearchView.get(); } IFeedView::SP getFeedView() const override { return _iFeedView.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 5a7afcc584d..527eaff07cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -417,7 +417,6 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) bool updateOk = _metaStore.updateMetaData(updOp.getLid(), updOp.getBucketId(), updOp.getTimestamp()); assert(updateOk); (void) updateOk; - _metaStore.commit(CommitParam(serialNum)); } auto onWriteDone = createUpdateDoneContext(std::move(token), get_pending_lid_token(updOp), updOp.getUpdate()); @@ -605,7 +604,6 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & gate.await(); removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED); } - _metaStore.commit(CommitParam(serialNum)); } } @@ -621,9 +619,6 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo const SerialNum serialNum = op.getSerialNum(); const LidVectorContext::SP &ctx = op.getLidsToRemove(_params._subDbId); if (!ctx) { - if (useDocumentMetaStore(serialNum)) { - _metaStore.commit(CommitParam(serialNum)); - } return 0; } const LidVector &lidsToRemove(ctx->getLidVector()); @@ -634,7 +629,6 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo _gidToLidChangeHandler.notifyRemoves(std::make_shared<vespalib::GateCallback>(gate), gidsToRemove, serialNum); gate.await(); _metaStore.removeBatch(lidsToRemove, ctx->getDocIdLimit()); - _metaStore.commit(CommitParam(serialNum)); _lidReuseDelayer.delayReuse(lidsToRemove); } std::shared_ptr<vespalib::IDestructorCallback> onWriteDone; diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index ff75a59c41b..7982e8a8414 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -33,13 +33,6 @@ uint32_t calculateIndexingThreads(const ProtonConfig::Indexing & indexing, double concurrency, const HwInfo::Cpu &cpuInfo) { double scaledCores = cpuInfo.cores() * concurrency; - if (indexing.optimize != ProtonConfig::Indexing::Optimize::ADAPTIVE) { - // We are capping at 12 threads to reduce cost of waking up threads - // to achieve a better throughput. - // TODO: Fix this in a simpler/better way. - scaledCores = std::min(12.0, scaledCores); - } - uint32_t indexingThreads = std::max((int32_t)std::ceil(scaledCores / 3), indexing.threads); return std::max(indexingThreads, 1u); } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 5cc0ac5a186..03ddcf3605b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -58,7 +58,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB { return IReprocessingTask::List(); } - void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) override { } + void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &, OnDone) override { } ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); } IFeedView::SP getFeedView() const override { return IFeedView::SP(); } void clearViews() override {} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index abc8eb679dd..743cd9af8fc 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -87,7 +87,7 @@ public: return _writables; } void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor>) const override { } - void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>) const override { } + void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>, OnDone) const override { } ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &) const override { return ExclusiveAttributeReadAccessor::UP(); diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h index 6b08eecf61f..0c6af0a4d25 100644 --- a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h +++ b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h @@ -7,14 +7,35 @@ namespace proton::test { /** + * Run the given function in the master thread and wait until all threads done. + */ +template <typename FunctionType> +void +runInMasterAndSyncAll(searchcorespi::index::IThreadingService &writeService, FunctionType func) +{ + writeService.master().execute(vespalib::makeLambdaTask(std::move(func))); + writeService.sync_all_executors(); +} + +/** * Run the given function in the master thread and wait until done. */ template <typename FunctionType> void +runInMasterAndSync(searchcorespi::index::IThreadingService &writeService, FunctionType func) +{ + writeService.master().execute(vespalib::makeLambdaTask(std::move(func))); + writeService.master().sync(); +} + +/** + * Run the given function in the master thread. + */ +template <typename FunctionType> +void runInMaster(searchcorespi::index::IThreadingService &writeService, FunctionType func) { writeService.master().execute(vespalib::makeLambdaTask(std::move(func))); - writeService.sync_all_executors(); } } diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index bafcd545e33..c077ab83a6e 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -200,9 +200,9 @@ struct ReferenceAttributeTest : public ::testing::Test { void notifyReferencedRemove(const GlobalId &gid) { _attr->notifyReferencedRemove(gid); } - void setGidToLidMapperFactory(std::shared_ptr<MyGidToLidMapperFactory> factory) { + void setGidToLidMapperFactory(std::shared_ptr<MyGidToLidMapperFactory> factory, const std::vector<GlobalId>& removes) { _attr->setGidToLidMapperFactory(factory); - _attr->populateTargetLids(); + _attr->populateTargetLids(removes); } uint32_t getUniqueGids() { return getStatus().getNumUniqueValues(); @@ -257,7 +257,7 @@ TEST_F(ReferenceAttributeTest, reference_for_a_document_can_be_cleared) TEST_F(ReferenceAttributeTest, lid_beyond_range_is_mapped_to_zero) { auto factory = std::make_shared<MyGidToLidMapperFactory>(); - setGidToLidMapperFactory(factory); + setGidToLidMapperFactory(factory, {}); ensureDocIdLimit(5); _attr->addDocs(1); set(5, toGid(doc2)); @@ -318,7 +318,7 @@ TEST_F(ReferenceAttributeTest, update_uses_gid_mapper_to_set_target_lid) { ensureDocIdLimit(6); auto factory = std::make_shared<MyGidToLidMapperFactory>(); - setGidToLidMapperFactory(factory); + setGidToLidMapperFactory(factory, {}); set(1, toGid(doc1)); set(2, toGid(doc2)); set(4, toGid(doc1)); @@ -371,7 +371,7 @@ void checkPopulateTargetLids(ReferenceAttributeTest &f) { auto factory = std::make_shared<MyGidToLidMapperFactory>(); - f.setGidToLidMapperFactory(factory); + f.setGidToLidMapperFactory(factory, {}); f.assertTargetLid(1, 10); f.assertTargetLid(2, 17); f.assertTargetLid(3, 10); @@ -401,6 +401,22 @@ TEST_F(ReferenceAttributeTest, populateTargetLids_uses_gid_mapper_to_update_lid_ EXPECT_TRUE(vespalib::unlink("test.udat")); } +TEST_F(ReferenceAttributeTest, populateTargetLids_handles_removes) +{ + preparePopulateTargetLids(*this); + auto factory = std::make_shared<MyGidToLidMapperFactory>(); + setGidToLidMapperFactory(factory, { toGid(doc1) }); + assertTargetLid(1, 0); + assertTargetLid(2, 17); + assertTargetLid(3, 0); + assertTargetLid(4, 0); + assertNoTargetLid(5); + assertLids(0, { }); + assertLids(10, { }); + assertLids(17, { 2 }); + assertLids(18, { }); +} + TEST_F(ReferenceAttributeTest, notifyReferencedPut_and_notifyReferencedRemove_changes_reverse_mapping) { preparePopulateTargetLids(*this); diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 342c1ea568b..eb822313d61 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -366,8 +366,8 @@ ReferenceAttribute::notifyReferencedPut(const GlobalId &gid, DocId targetLid) commit(); } -void -ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid) +bool +ReferenceAttribute::notifyReferencedRemoveNoCommit(const GlobalId &gid) { EntryRef ref = _store.find(gid); if (ref.valid()) { @@ -377,6 +377,15 @@ ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid) if (oldTargetLid != 0) { _store.remove(ref); } + return true; + } + return false; +} + +void +ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid) +{ + if (notifyReferencedRemoveNoCommit(gid)) { commit(); } } @@ -401,7 +410,7 @@ public: } void -ReferenceAttribute::populateTargetLids() +ReferenceAttribute::populateTargetLids(const std::vector<GlobalId>& removes) { if (_gidToLidMapperFactory) { std::unique_ptr<IGidToLidMapper> mapperUP = _gidToLidMapperFactory->getMapper(); @@ -409,6 +418,9 @@ ReferenceAttribute::populateTargetLids() TargetLidPopulator populator(*this); mapper.foreach(populator); } + for (auto& remove : removes) { + notifyReferencedRemoveNoCommit(remove); + } commit(); } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index b411d6cb923..4016230ef89 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -84,8 +84,9 @@ public: void notifyReferencedPutNoCommit(const GlobalId &gid, DocId targetLid); void notifyReferencedPut(const GlobalId &gid, DocId targetLid); + bool notifyReferencedRemoveNoCommit(const GlobalId &gid); void notifyReferencedRemove(const GlobalId &gid); - void populateTargetLids(); + void populateTargetLids(const std::vector<GlobalId>& removes); void clearDocs(DocId lidLow, DocId lidLimit) override; void onShrinkLidSpace() override; diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index ee9a97a4685..52a363c9888 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchsummary/docsummary/positionsdfw.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/searchsummary/docsummary/docsumstate.h> +#include <vespa/searchsummary/test/slime_value.h> #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> @@ -113,7 +114,7 @@ struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { template <typename AttrType> void checkWritePositionField(Test &test, AttrType &attr, - uint32_t doc_id, const string &expected) { + uint32_t doc_id, const vespalib::string &expect_json) { for (AttributeVector::DocId i = 0; i < doc_id + 1; ) { attr.addDoc(i); if (i == 007) { @@ -133,7 +134,7 @@ void checkWritePositionField(Test &test, AttrType &attr, PositionsDFW::UP writer = createPositionsDFW(attr.getName().c_str(), &attribute_man); ASSERT_TRUE(writer.get()); - ResType res_type = RES_LONG_STRING; + ResType res_type = RES_JSONSTRING; MyGetDocsumsStateCallback callback; GetDocsumsState state(callback); state._attributes.push_back(&attr); @@ -142,19 +143,17 @@ void checkWritePositionField(Test &test, AttrType &attr, vespalib::slime::SlimeInserter inserter(target); writer->insertField(doc_id, &state, res_type, inserter); - vespalib::Memory got = target.get().asString(); - test.EXPECT_EQUAL(expected.size(), got.size); - test.EXPECT_EQUAL(expected, string(got.data, got.size)); + test::SlimeValue expected(expect_json); + test.EXPECT_EQUAL(expected.slime, target); } void Test::requireThat2DPositionFieldIsWritten() { SingleInt64ExtAttribute attr("foo"); - checkWritePositionField(*this, attr, 0x3e, "<position x=\"6\" y=\"7\" latlong=\"N0.000007;E0.000006\" />"); - checkWritePositionField(*this, attr, 007, "<position x=\"-1\" y=\"-1\" latlong=\"S0.000001;W0.000001\" />"); - checkWritePositionField(*this, attr, 0x42, "<position x=\"0\" y=\"-1\" latlong=\"S0.000001;E0.000000\" />"); - checkWritePositionField(*this, attr, 0x17, "<position x=\"-16711935\" y=\"16711935\" latlong=\"N16.711935;W16.711935\" />"); - checkWritePositionField(*this, attr, 42, ""); - + checkWritePositionField(*this, attr, 0x3e, "{x:6,y:7,latlong:'N0.000007;E0.000006'}"); + checkWritePositionField(*this, attr, 007, "{x:-1,y:-1,latlong:'S0.000001;W0.000001'}"); + checkWritePositionField(*this, attr, 0x42, "{x:0,y:-1,latlong:'S0.000001;E0.000000'}"); + checkWritePositionField(*this, attr, 0x17, "{x:-16711935,y:16711935,latlong:'N16.711935;W16.711935'}"); + checkWritePositionField(*this, attr, 42, "null"); } } // namespace diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 0a421882784..5855dd0e8fd 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -65,8 +65,6 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { strlen("long_string"))); EXPECT_TRUE(packer->AddLongData( "long_data", strlen("long_data"))); - EXPECT_TRUE(packer->AddLongString( "xml_string", - strlen("xml_string"))); FieldBlock jsf1("{foo:1, bar:2}"); EXPECT_TRUE(packer->AddLongData(jsf1.data(), jsf1.dataLen())); EXPECT_TRUE(packer->AddLongString("abc", 3)); @@ -98,7 +96,6 @@ DocsumFixture::DocsumFixture() EXPECT_TRUE(cfg->AddConfigEntry("data_field", RES_DATA)); EXPECT_TRUE(cfg->AddConfigEntry("longstring_field", RES_LONG_STRING)); EXPECT_TRUE(cfg->AddConfigEntry("longdata_field", RES_LONG_DATA)); - EXPECT_TRUE(cfg->AddConfigEntry("xmlstring_field", RES_XMLSTRING)); EXPECT_TRUE(cfg->AddConfigEntry("jsonstring_field", RES_JSONSTRING)); EXPECT_TRUE(cfg->AddConfigEntry("bad_jsonstring_field", RES_JSONSTRING)); config->CreateEnumMaps(); @@ -121,7 +118,6 @@ TEST_FF("require that docsum can be written as slime", DocsumFixture(), Slime()) EXPECT_EQUAL(f2.get()["data_field"].asData().make_string(), std::string("data")); EXPECT_EQUAL(f2.get()["longstring_field"].asString().make_string(), std::string("long_string")); EXPECT_EQUAL(f2.get()["longdata_field"].asData().make_string(), std::string("long_data")); - EXPECT_EQUAL(f2.get()["xmlstring_field"].asString().make_string(), std::string("xml_string")); EXPECT_EQUAL(f2.get()["jsonstring_field"]["foo"].asLong(), 1u); EXPECT_EQUAL(f2.get()["jsonstring_field"]["bar"].asLong(), 2u); EXPECT_EQUAL(f2.get()["bad_jsonstring_field"].type().getId(), 0u); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index 6dd952f87ef..448feedac80 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -118,7 +118,6 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type } break; case RES_JSONSTRING: - case RES_XMLSTRING: case RES_FEATUREDATA: case RES_LONG_STRING: case RES_STRING: { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index afc1ec4b3f2..fada441f718 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -121,7 +121,6 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *s target.insertLong(valint64); break; } - case RES_XMLSTRING: case RES_JSONSTRING: case RES_FEATUREDATA: case RES_LONG_STRING: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 766ebdc51f4..666c5098ca4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -120,7 +120,6 @@ static void convertEntry(GetDocsumsState *state, case RES_STRING: case RES_LONG_STRING: case RES_FEATUREDATA: - case RES_XMLSTRING: entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); if (len != 0) { inserter.insertString(Memory(ptr, len)); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index 3c01c23c965..ceb358e5859 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -255,7 +255,6 @@ GeneralResult::unpack(const char *buf, const size_t buflen) break; } - case RES_XMLSTRING: case RES_JSONSTRING: case RES_FEATUREDATA: case RES_LONG_STRING: { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 43a3dc2163b..ce14069a135 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -94,15 +94,10 @@ AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type vespalib::string value = vespalib::stringify(absdist); vespalib::Memory data(value.c_str(), value.size()); - if (type == RES_STRING || - type == RES_LONG_STRING || - type == RES_XMLSTRING) - { + if (type == RES_STRING || type == RES_LONG_STRING) { target.insertString(data); } - if (type == RES_LONG_DATA || - type == RES_DATA) - { + if (type == RES_LONG_DATA || type == RES_DATA) { target.insertData(data); } } @@ -173,63 +168,21 @@ insertFromAttr(const attribute::IAttributeVector &attribute, uint32_t docid, ves } } -vespalib::asciistream -formatField(const attribute::IAttributeVector &attribute, uint32_t docid, ResType type) { - vespalib::asciistream target; - int32_t docx = 0; - int32_t docy = 0; - - IntegerContent pos; - pos.fill(attribute, docid); - uint32_t numValues = pos.size(); - LOG(debug, "docid=%d, numValues=%d", docid, numValues); - - bool isShort = !IDocsumFieldWriter::IsBinaryCompatible(type, RES_LONG_STRING); - for (uint32_t i = 0; i < numValues; i++) { - int64_t docxy(pos[i]); - vespalib::geo::ZCurve::decode(docxy, &docx, &docy); - if (docx == 0 && docy == INT_MIN) { - LOG(spam, "skipping empty zcurve value"); - continue; - } - double degrees_ns = docy; - degrees_ns /= 1000000.0; - double degrees_ew = docx; - degrees_ew /= 1000000.0; - - target << "<position x=\"" << docx << "\" y=\"" << docy << "\""; - target << " latlong=\""; - target << vespalib::FloatSpec::fixed; - if (degrees_ns < 0) { - target << "S" << (-degrees_ns); - } else { - target << "N" << degrees_ns; - } - target << ";"; - if (degrees_ew < 0) { - target << "W" << (-degrees_ew); - } else { - target << "E" << degrees_ew; - } - target << "\" />"; - if (isShort && target.size() > 30000) { - target << "<overflow />"; - break; - } - } - return target; -} +void checkExpected(ResType type) { + static bool alreadyWarned = false; + if (type == RES_JSONSTRING) return; + if (alreadyWarned) return; + alreadyWarned = true; + LOG(error, "Unexpected summary field type %s", ResultConfig::GetResTypeName(type)); } +} // namespace + void PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target) { - if (type == RES_XMLSTRING) { - insertFromAttr(get_attribute(*dsState), docid, target); - return; - } - vespalib::asciistream val(formatField(get_attribute(*dsState), docid, type)); - target.insertString(vespalib::Memory(val.c_str(), val.size())); + checkExpected(type); + insertFromAttr(get_attribute(*dsState), docid, target); } //-------------------------------------------------------------------------- diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 97435d793be..858fdea2404 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -27,7 +27,6 @@ enum ResType { RES_DATA, RES_LONG_STRING, RES_LONG_DATA, - RES_XMLSTRING, RES_JSONSTRING, RES_TENSOR, RES_FEATUREDATA diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 09604a7316c..d3c0caeec48 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -53,7 +53,6 @@ ResultConfig::GetResTypeName(ResType type) case RES_DATA: return "data"; case RES_LONG_STRING: return "longstring"; case RES_LONG_DATA: return "longdata"; - case RES_XMLSTRING: return "xmlstring"; case RES_JSONSTRING: return "jsonstring"; case RES_TENSOR: return "tensor"; case RES_FEATUREDATA: return "featuredata"; @@ -126,6 +125,8 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const Reset(); int maxclassID = 0x7fffffff; // avoid negative classids _defaultSummaryId = cfg.defaultsummaryid; + _useV8geoPositions = cfg.usev8geopositions; + for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) { const auto& cfg_class = cfg.classes[i]; if (cfg_class.name.empty()) { @@ -173,7 +174,7 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const } else if (strcmp(fieldtype, "longdata") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_LONG_DATA); } else if (strcmp(fieldtype, "xmlstring") == 0) { - rc = resClass->AddConfigEntry(fieldname, RES_XMLSTRING); + rc = resClass->AddConfigEntry(fieldname, RES_JSONSTRING); } else if (strcmp(fieldtype, "jsonstring") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_JSONSTRING); } else if (strcmp(fieldtype, "tensor") == 0) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 4cc37512929..8a8bfabaaec 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -33,6 +33,7 @@ private: typedef vespalib::hash_map<vespalib::string, uint32_t> NameMap; typedef vespalib::hash_map<uint32_t, ResultClass::UP> IdMap; uint32_t _defaultSummaryId; + bool _useV8geoPositions; search::util::StringEnum _fieldEnum; IdMap _classLookup; NameMap _nameLookup; // name -> class id @@ -41,6 +42,7 @@ private: void Init(); public: + bool useV8geoPositions() const { return _useV8geoPositions; } class iterator { public: iterator(IdMap::iterator it) : _it(it) { } @@ -123,11 +125,10 @@ public: return (b == RES_STRING || b == RES_DATA); case RES_LONG_STRING: case RES_LONG_DATA: - case RES_XMLSTRING: case RES_FEATUREDATA: case RES_JSONSTRING: return (b == RES_LONG_STRING || b == RES_LONG_DATA || - b == RES_XMLSTRING || b == RES_FEATUREDATA || b == RES_JSONSTRING); + b == RES_FEATUREDATA || b == RES_JSONSTRING); default: return false; } @@ -159,9 +160,8 @@ public: return b == RES_INT64; case RES_STRING: case RES_LONG_STRING: - case RES_XMLSTRING: case RES_JSONSTRING: - return (b == RES_STRING || b == RES_LONG_STRING || b == RES_XMLSTRING || b == RES_JSONSTRING); + return (b == RES_STRING || b == RES_LONG_STRING || b == RES_JSONSTRING); case RES_DATA: case RES_LONG_DATA: return (b == RES_DATA || b == RES_LONG_DATA); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp index 6c2c4f9c4ea..4ab06cbd41a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp @@ -114,7 +114,6 @@ ResultPacker::AddEmpty() case RES_INT64: return AddInt64(search::attribute::getUndefined<int64_t>()); case RES_STRING: return AddString(nullptr, 0); case RES_DATA: return AddData(nullptr, 0); - case RES_XMLSTRING: case RES_JSONSTRING: case RES_FEATUREDATA: case RES_LONG_STRING: return AddLongString(nullptr, 0); diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h index 3226c16aba3..4b489d41f7d 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.h +++ b/storage/src/tests/distributor/distributor_stripe_test_util.h @@ -26,7 +26,7 @@ class DocumentSelectionParser; class ExternalOperationHandler; class IdealStateManager; class IdealStateMetricSet; -class NodeSupportedFeatures; +struct NodeSupportedFeatures; class Operation; class StripeBucketDBUpdater; diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 02b43a32df3..1c47170de6c 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -106,12 +106,14 @@ PersistenceProviderWrapper::putAsync(const spi::Bucket& bucket, spi::Timestamp t } void -PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, spi::Timestamp timestamp, const spi::DocumentId& id, +PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, std::vector<TimeStampAndDocumentId> ids, spi::Context& context, spi::OperationComplete::UP onComplete) { - LOG_SPI("remove(" << bucket << ", " << timestamp << ", " << id << ")"); + for (const TimeStampAndDocumentId & stampedId : ids) { + LOG_SPI("remove(" << bucket << ", " << stampedId.first << ", " << stampedId.second << ")"); + } CHECK_ERROR_ASYNC(spi::RemoveResult, FAIL_REMOVE, onComplete); - _spi.removeAsync(bucket, timestamp, id, context, std::move(onComplete)); + _spi.removeAsync(bucket, std::move(ids), context, std::move(onComplete)); } void diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index cfc7002a643..1552a955221 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -100,7 +100,7 @@ public: spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override; spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; void putAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::Context&, spi::OperationComplete::UP) override; - void removeAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId> ids, spi::Context&, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&, spi::OperationComplete::UP) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override; diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index f3560bfa2cb..c163f6de024 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -130,19 +130,6 @@ PersistenceTestUtils::setupExecutor(uint32_t numThreads) { _sequenceTaskExecutor = vespalib::SequencedTaskExecutor::create(test_executor, numThreads, 1000, vespalib::Executor::OptimizeFor::ADAPTIVE); } -document::Document::SP -PersistenceTestUtils::schedulePut( - uint32_t location, - spi::Timestamp timestamp, - uint32_t minSize, - uint32_t maxSize) -{ - document::Document::SP doc(createRandomDocumentAtLocation(location, timestamp, minSize, maxSize)); - auto msg = std::make_shared<api::PutCommand>(makeDocumentBucket(document::BucketId(16, location)), doc, timestamp); - fsHandler().schedule(msg); - return doc; -} - StorBucketDatabase::WrappedEntry PersistenceTestUtils::getBucket(const document::BucketId& id) { diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index dd9f5f15123..fc986c3c6f2 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -108,14 +108,10 @@ public: PersistenceTestUtils(); ~PersistenceTestUtils() override; - document::Document::SP schedulePut(uint32_t location, spi::Timestamp timestamp, uint32_t minSize = 0, uint32_t maxSize = 128); - - void setupDisks(); void setupExecutor(uint32_t numThreads); void TearDown() override { if (_sequenceTaskExecutor) { - _sequenceTaskExecutor->sync_all(); _sequenceTaskExecutor.reset(); } _env.reset(); diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index 0c862f11b05..51fe4ef4bd3 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -41,11 +41,14 @@ TEST_F(ProcessAllHandlerTest, remove_location) { AsyncHandler handler(getEnv(), getPersistenceProvider(), _bucketOwnershipNotifier, *_sequenceTaskExecutor, _bucketIdFactory); auto tracker = handler.handleRemoveLocation(*cmd, createTracker(cmd, bucket)); + std::shared_ptr<api::StorageMessage> msg; + ASSERT_TRUE(_replySender.queue.getNext(msg, 60s)); + EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n", dumpBucket(bucketId)); - auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(std::move(*tracker).stealReplySP()); + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(msg); ASSERT_TRUE(reply); EXPECT_EQ(2u, reply->documents_removed()); } @@ -65,6 +68,9 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { auto cmd = std::make_shared<api::RemoveLocationCommand>("testdoctype1.headerval % 2 == 0", bucket); auto tracker = handler.handleRemoveLocation(*cmd, createTracker(cmd, bucket)); + std::shared_ptr<api::StorageMessage> msg; + ASSERT_TRUE(_replySender.queue.getNext(msg, 60s)); + EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n" "DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n" "DocEntry(102, 1, id:mail:testdoctype1:n=4:62608.html)\n" @@ -77,7 +83,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) { "DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n", dumpBucket(bucketId)); - auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(std::move(*tracker).stealReplySP()); + auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(msg); ASSERT_TRUE(reply); EXPECT_EQ(5u, reply->documents_removed()); } diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index aac2b0748c4..b5161673af3 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -413,20 +413,14 @@ AsyncHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, MessageTrack std::make_shared<document::DocIdOnly>(), processor, spi::NEWEST_DOCUMENT_ONLY,tracker->context()); - std::vector<std::future<std::unique_ptr<spi::Result>>> results; - results.reserve(to_remove.size()); - for (auto & entry : to_remove) { - auto catcher = std::make_unique<spi::CatchResult>(); - results.push_back(catcher->future_result()); - _spi.removeAsync(bucket, entry.first, entry.second, tracker->context(), std::move(catcher)); - } - for (auto & future : results) { - auto result = future.get(); - if (result->getErrorCode() != spi::Result::ErrorType::NONE) { - throw std::runtime_error(fmt("Failed to do remove for removelocation: %s", result->getErrorMessage().c_str())); - } - } - tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, to_remove.size())); + auto task = makeResultTask([&cmd, tracker = std::move(tracker), removed = to_remove.size()](spi::Result::UP response) { + tracker->checkForError(*response); + tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, removed)); + tracker->sendReply(); + }); + + _spi.removeAsync(bucket, std::move(to_remove), tracker->context(), + std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task))); return tracker; } diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 7d7c8b79e0c..254a26aa454 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -506,9 +506,10 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), std::move(docId), "put", _clock, _env._metrics.merge_handler_metrics.put_latency); _spi.putAsync(bucket, timestamp, std::move(doc), context, std::move(complete)); } else { - DocumentId docId(e._docName); - auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), docId, "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency); - _spi.removeAsync(bucket, timestamp, docId, context, std::move(complete)); + std::vector<spi::PersistenceProvider::TimeStampAndDocumentId> ids; + ids.emplace_back(timestamp, e._docName); + auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].second, "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency); + _spi.removeAsync(bucket, std::move(ids), context, std::move(complete)); } } diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 9ccd901744b..752c1175f22 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -154,11 +154,11 @@ ProviderErrorWrapper::putAsync(const spi::Bucket &bucket, spi::Timestamp ts, spi } void -ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, spi::Timestamp ts, const document::DocumentId &docId, +ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, std::vector<TimeStampAndDocumentId> ids, spi::Context & context, spi::OperationComplete::UP onComplete) { onComplete->addResultHandler(this); - _impl.removeAsync(bucket, ts, docId, context, std::move(onComplete)); + _impl.removeAsync(bucket, std::move(ids), context, std::move(onComplete)); } void diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 14d20cf8a52..7285c405d5c 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -58,7 +58,7 @@ public: void register_error_listener(std::shared_ptr<ProviderErrorListener> listener); void putAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentSP, spi::Context &, spi::OperationComplete::UP) override; - void removeAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; + void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId>, spi::Context&, spi::OperationComplete::UP) override; void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&, spi::OperationComplete::UP) override; void updateAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentUpdateSP, spi::Context &, spi::OperationComplete::UP) override; void setActiveStateAsync(const spi::Bucket& b, spi::BucketInfo::ActiveState newState, spi::OperationComplete::UP onComplete) override; diff --git a/vespa-feed-client/abi-spec.json b/vespa-feed-client/abi-spec.json index 808fe152fee..7f78e81b447 100644 --- a/vespa-feed-client/abi-spec.json +++ b/vespa-feed-client/abi-spec.json @@ -195,6 +195,7 @@ "public" ], "methods": [ + "public void <init>(java.time.Duration)", "public void <init>(java.time.Duration, java.time.Duration)", "public void success()", "public void failure(ai.vespa.feed.client.HttpResponse)", diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index cdf55f0ba7e..3b79d47b494 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -37,7 +37,7 @@ public class FeedClientBuilder { int connectionsPerEndpoint = 4; int maxStreamsPerConnection = 4096; FeedClient.RetryStrategy retryStrategy = defaultRetryStrategy; - FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Duration.ofSeconds(10), Duration.ofMinutes(10)); + FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Duration.ofSeconds(10)); Path certificateFile; Path privateKeyFile; Path caCertificatesFile; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java index b878840d70f..cb5e35c79a5 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java @@ -31,20 +31,32 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { private final long graceMillis; private final long doomMillis; + /** + * Creates a new circuit breaker with the given grace periods. + * @param grace the period of consecutive failures before state changes to half-open. + */ + public GracePeriodCircuitBreaker(Duration grace) { + this(System::currentTimeMillis, grace, null); + } + + /** + * Creates a new circuit breaker with the given grace periods. + * @param grace the period of consecutive failures before state changes to half-open. + * @param doom the period of consecutive failures before shutting down. + */ public GracePeriodCircuitBreaker(Duration grace, Duration doom) { this(System::currentTimeMillis, grace, doom); + if (doom.isNegative()) + throw new IllegalArgumentException("Doom delay must be non-negative"); } GracePeriodCircuitBreaker(LongSupplier clock, Duration grace, Duration doom) { if (grace.isNegative()) throw new IllegalArgumentException("Grace delay must be non-negative"); - if (doom.isNegative()) - throw new IllegalArgumentException("Doom delay must be non-negative"); - this.clock = requireNonNull(clock); this.graceMillis = grace.toMillis(); - this.doomMillis = doom.toMillis(); + this.doomMillis = doom == null ? -1 : doom.toMillis(); } @Override @@ -74,11 +86,11 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { long failingMillis = clock.getAsLong() - failingSinceMillis.get(); if (failingMillis > graceMillis && halfOpen.compareAndSet(false, true)) log.log(INFO, "Circuit breaker is now half-open, as no requests have succeeded for the " + - "last " + failingMillis + "ms. The server will be pinged to see if it recovers, " + - "but this client will give up if no successes are observed within " + doomMillis + "ms. " + - "First failure was '" + detail.get() + "'."); + "last " + failingMillis + "ms. The server will be pinged to see if it recovers" + + (doomMillis >= 0 ? ", but this client will give up if no successes are observed within " + doomMillis + "ms" : "") + + ". First failure was '" + detail.get() + "'."); - if (failingMillis > doomMillis && open.compareAndSet(false, true)) + if (doomMillis >= 0 && failingMillis > doomMillis && open.compareAndSet(false, true)) log.log(WARNING, "Circuit breaker is now open, after " + doomMillis + "ms of failing request, " + "and this client will give up and abort its remaining feed operations."); diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java index 11b844f2c69..0f840201ca8 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java @@ -208,16 +208,16 @@ class HttpRequestStrategyTest { // Enqueue some operations to the same id, which are serialised, and then shut down while operations are in flight. Phaser phaser = new Phaser(2); Phaser blocker = new Phaser(2); - AtomicReference<CompletableFuture<HttpResponse>> completion = new AtomicReference<>(); cluster.expect((req, vessel) -> { if (req == blocking) { - phaser.arriveAndAwaitAdvance(); // Synchronise with tst main thread, and then ... + phaser.arriveAndAwaitAdvance(); // Synchronise with test main thread, and then ... blocker.arriveAndAwaitAdvance(); // ... block dispatch thread, so we get something in the queue. throw new RuntimeException("armageddon"); // Dispatch thread should die, tearing down everything. } else if (req == failing) { phaser.arriveAndAwaitAdvance(); // Let test thread enqueue more ops before failing (and retrying) this. vessel.completeExceptionally(new IOException("failed")); + phaser.arriveAndAwaitAdvance(); // Ensure a retry is scheduled before test thread is allowed to continue. } else phaser.arriveAndAwaitAdvance(); // Don't complete from mock cluster, but require destruction to do this. }); @@ -228,7 +228,8 @@ class HttpRequestStrategyTest { CompletableFuture<HttpResponse> blocked = strategy.enqueue(id3, blocking); CompletableFuture<HttpResponse> delayed = strategy.enqueue(id4, request); phaser.arriveAndAwaitAdvance(); // inflight completes dispatch, but causes no response. - phaser.arriveAndAwaitAdvance(); // failed completes dispatch, and a retry is enqueued. + phaser.arriveAndAwaitAdvance(); // failed is allowed to dispatch ... + phaser.arriveAndAwaitAdvance(); // ... and a retry is enqueued. phaser.arriveAndAwaitAdvance(); // blocked starts dispatch, and hangs, blocking dispatch thread. // Current state: inflight is "inflight to cluster", serialised1/2 are waiting completion of it; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java index c877daeeb2a..98b6aee1e33 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java @@ -67,10 +67,10 @@ class EndpointResultQueue { InflightOperation operation = inflightOperations.remove(result.getOperationId()); if (operation == null) { if (duplicateGivesWarning) { - log.warning("Result for ID '" + result.getOperationId() + "' received from '" + endpoint + - "', but we have no record of a sent operation. Either something is wrong on the server side " + - "(bad VIP usage?), or we have somehow received duplicate results, " + - "or operation was received _after_ client-side timeout."); + log.info("Result for ID '" + result.getOperationId() + "' received from '" + endpoint + + "', but we have no record of a sent operation. This may happen if an operation is " + + "initiated, but also retried, due to HTTP failure. Otherwise, something is wrong on " + + "the server side (bad VIP usage?), or operation was received _after_ client-side timeout."); } return; } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java index 831ec24ac5f..1f9216192fc 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -76,7 +76,8 @@ public class VespaCliTestRunner implements TestRunner { void runTests(Suite suite, byte[] config) { Process process = null; try { - process = testRunProcessBuilder(suite, toEndpointsConfig(config)).start(); + TestConfig testConfig = TestConfig.fromJson(config); + process = testRunProcessBuilder(suite, testConfig).start(); BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); in.lines().forEach(line -> { if (line.length() > 1 << 13) @@ -95,14 +96,17 @@ public class VespaCliTestRunner implements TestRunner { } } - ProcessBuilder testRunProcessBuilder(Suite suite, String endpointsConfig) { + ProcessBuilder testRunProcessBuilder(Suite suite, TestConfig config) throws IOException { Path suitePath = getChildDirectory(artifactsPath, "tests") .flatMap(testsPath -> getChildDirectory(testsPath, toSuiteDirectoryName(suite))) .orElseThrow(() -> new IllegalStateException("No tests found, for suite '" + suite + "'")); - ProcessBuilder builder = new ProcessBuilder("vespa", "test", "--endpoints", endpointsConfig); + ProcessBuilder builder = new ProcessBuilder("vespa", "test", suitePath.toAbsolutePath().toString(), + "--application", config.application().toFullString(), + "--endpoints", toEndpointsConfig(config), + "--data-plane-public-cert", artifactsPath.resolve("cert").toAbsolutePath().toString(), + "--data-plane-private-key", artifactsPath.resolve("key").toAbsolutePath().toString()); builder.redirectErrorStream(true); - builder.directory(suitePath.toFile()); return builder; } @@ -133,8 +137,7 @@ public class VespaCliTestRunner implements TestRunner { } } - static String toEndpointsConfig(byte[] testConfig) throws IOException { - TestConfig config = TestConfig.fromJson(testConfig); + static String toEndpointsConfig(TestConfig config) throws IOException { Cursor root = new Slime().setObject(); Cursor endpointsArray = root.setArray("endpoints"); config.deployments().get(config.zone()).forEach((cluster, url) -> { diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java index 68d44a386f8..be00f28d2c9 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; +import ai.vespa.hosted.api.TestConfig; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -19,34 +20,28 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ class VespaCliTestRunnerTest { - @Test - void testEndpointsConfig() throws IOException { - byte[] testConfig = ("{\n" + - " \"application\": \"t:a:i\",\n" + - " \"zone\": \"dev.aws-us-east-1c\",\n" + - " \"system\": \"publiccd\",\n" + - " \"isCI\": true,\n" + - " \"zoneEndpoints\": {\n" + - " \"dev.aws-us-east-1c\": {\n" + - " \"default\": \"https://dev.endpoint:443/\"\n" + - " },\n" + - " \"prod.aws-us-east-1a\": {\n" + - " \"default\": \"https://prod.endpoint:443/\"\n" + - " }\n" + - " },\n" + - " \"clusters\": {\n" + - " \"prod.aws-us-east-1c\": [\n" + - " \"documents\"\n" + - " ]\n" + - " }\n" + - "}\n").getBytes(StandardCharsets.UTF_8); - - assertEquals("{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}", - VespaCliTestRunner.toEndpointsConfig(testConfig)); - } + static final TestConfig testConfig = TestConfig.fromJson(("{\n" + + " \"application\": \"t:a:i\",\n" + + " \"zone\": \"dev.aws-us-east-1c\",\n" + + " \"system\": \"publiccd\",\n" + + " \"isCI\": true,\n" + + " \"zoneEndpoints\": {\n" + + " \"dev.aws-us-east-1c\": {\n" + + " \"default\": \"https://dev.endpoint:443/\"\n" + + " },\n" + + " \"prod.aws-us-east-1a\": {\n" + + " \"default\": \"https://prod.endpoint:443/\"\n" + + " }\n" + + " },\n" + + " \"clusters\": {\n" + + " \"prod.aws-us-east-1c\": [\n" + + " \"documents\"\n" + + " ]\n" + + " }\n" + + "}\n").getBytes(StandardCharsets.UTF_8)); @Test - void testSuitePathDiscovery() throws IOException { + void testSetup() throws IOException { Path temp = Files.createTempDirectory("vespa-cli-test-runner-test-"); temp.toFile().deleteOnExit(); VespaCliTestRunner runner = new VespaCliTestRunner(temp); @@ -54,14 +49,18 @@ class VespaCliTestRunnerTest { Path tests = Files.createDirectory(temp.resolve("tests")); assertTrue(runner.isSupported()); - IllegalStateException expected = assertThrows(IllegalStateException.class, - () -> runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, "")); - assertEquals("No tests found, for suite 'SYSTEM_TEST'", expected.getMessage()); + IllegalStateException ise = assertThrows(IllegalStateException.class, + () -> runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, testConfig)); + assertEquals("No tests found, for suite 'SYSTEM_TEST'", ise.getMessage()); Path systemTests = Files.createDirectory(tests.resolve("system-test")); - ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, "config"); - assertEquals(systemTests.toFile(), builder.directory()); - assertEquals(List.of("vespa", "test", "--endpoints", "config"), builder.command()); + ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, testConfig); + assertEquals(List.of("vespa", "test", systemTests.toAbsolutePath().toString(), + "--application", "t.a.i", + "--endpoints", "{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}", + "--data-plane-public-cert", temp.resolve("cert").toAbsolutePath().toString(), + "--data-plane-private-key", temp.resolve("key").toAbsolutePath().toString()), + builder.command()); } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 11fd97f2a1d..3faafd7b2e5 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -1,21 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; -import java.util.Set; -import java.util.logging.Logger; - import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.documentapi.metrics.DocumentOperationStatus; import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.jdisc.Metric; -import java.util.logging.Level; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Trace; import com.yahoo.vespa.http.client.core.ErrorCode; import com.yahoo.vespa.http.client.core.OperationStatus; +import java.util.logging.Level; +import java.util.logging.Logger; + /** * Catch message bus replies and make the available to a given session. * @@ -42,20 +41,26 @@ public class FeedReplyReader implements ReplyHandler { final double latencyInSeconds = (System.currentTimeMillis() - context.creationTime) / 1000.0d; metric.set(MetricNames.LATENCY, latencyInSeconds, null); - if (reply.hasErrors()) { - Set<Integer> errorCodes = reply.getErrorCodes(); - metricsHelper.reportFailure(DocumentOperationType.fromMessage(reply.getMessage()), - DocumentOperationStatus.fromMessageBusErrorCodes(errorCodes)); + DocumentOperationType type = DocumentOperationType.fromMessage(reply.getMessage()); + boolean conditionMet = conditionMet(reply); + if (reply.hasErrors() && conditionMet) { + DocumentOperationStatus status = DocumentOperationStatus.fromMessageBusErrorCodes(reply.getErrorCodes()); + metricsHelper.reportFailure(type, status); metric.add(MetricNames.FAILED, 1, null); - enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, - reply.getError(0).getCode() == DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED, reply.getTrace()); + enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, false, reply.getTrace()); } else { - metricsHelper.reportSuccessful(DocumentOperationType.fromMessage(reply.getMessage()), latencyInSeconds); + metricsHelper.reportSuccessful(type, latencyInSeconds); metric.add(MetricNames.SUCCEEDED, 1, null); - enqueue(context, "Document processed.", ErrorCode.OK, false, reply.getTrace()); + if (!conditionMet) + metric.add(MetricNames.TEST_AND_SET_CONDITION_NOT_MET, 1, null); + enqueue(context, "Document processed.", ErrorCode.OK, !conditionMet, reply.getTrace()); } } + private static boolean conditionMet(Reply reply) { + return !reply.hasErrors() || reply.getError(0).getCode() != DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED; + } + private void enqueue(ReplyContext context, String message, ErrorCode status, boolean isConditionNotMet, Trace trace) { try { String traceMessage = (trace != null && trace.getLevel() > 0) ? trace.toString() : ""; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java index 8cd628a83d9..4b49e3594f8 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java @@ -18,6 +18,7 @@ public final class MetricNames { public static final String OPERATIONS_PER_SEC = PREFIX + "ops_per_sec"; public static final String LATENCY = PREFIX + "latency"; public static final String FAILED = PREFIX + "failed"; + public static final String TEST_AND_SET_CONDITION_NOT_MET = PREFIX + "test_and_set_condition_not_met"; public static final String PARSE_ERROR = PREFIX + "parse_error"; public static final String SUCCEEDED = PREFIX + "succeeded"; public static final String PENDING = PREFIX + "pending"; diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 2e2728781f4..964978e5510 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -705,6 +705,58 @@ TEST(DataStoreTest, control_static_sizes) { EXPECT_EQ(0, bs.size()); } +namespace { + +void test_free_element_to_held_buffer(bool direct, bool before_hold_buffer) +{ + MyStore s; + auto ref = s.addEntry(1); + EXPECT_EQ(0u, MyRef(ref).bufferId()); + s.switch_primary_buffer(); + EXPECT_EQ(1u, s.primary_buffer_id()); + + if (before_hold_buffer) { + if (direct) { + s.freeElem(ref, 1); + } else { + s.holdElem(ref, 1); + } + } + s.holdBuffer(0); // hold last buffer + if (!before_hold_buffer) { + if (direct) { + ASSERT_DEATH({ s.freeElem(ref, 1); }, "state.isOnHold\\(\\) && was_held"); + } else { + ASSERT_DEATH({ s.holdElem(ref, 1); }, "state.isActive\\(\\)"); + } + } + s.transferHoldLists(100); + s.trimHoldLists(101); +} + +} + +TEST(DataStoreTest, free_to_active_then_held_buffer_is_ok) +{ + test_free_element_to_held_buffer(true, true); +} + +TEST(DataStoreTest, hold_to_active_then_held_buffer_is_ok) +{ + test_free_element_to_held_buffer(false, true); +} + +#ifndef NDEBUG +TEST(DataStoreDeathTest, free_to_held_buffer_is_not_ok) +{ + test_free_element_to_held_buffer(true, false); +} + +TEST(DataStoreDeathTest, hold_to_held_buffer_is_not_ok) +{ + test_free_element_to_held_buffer(false, false); +} +#endif } diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h index d8bb0fb3d0f..be74c2a60d5 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.h +++ b/vespalib/src/vespa/vespalib/datastore/datastore.h @@ -27,6 +27,8 @@ template <typename RefT = EntryRefT<22> > class DataStoreT : public DataStoreBase { private: + void free_elem_internal(EntryRef ref, size_t numElems, bool was_held); + public: typedef RefT RefType; @@ -49,7 +51,7 @@ public: /** * Free element(s). */ - void freeElem(EntryRef ref, size_t numElems); + void freeElem(EntryRef ref, size_t numElems) { free_elem_internal(ref, numElems, false); } /** * Hold element(s). diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.hpp b/vespalib/src/vespa/vespalib/datastore/datastore.hpp index 81027196a75..23dcb9222a1 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.hpp +++ b/vespalib/src/vespa/vespalib/datastore/datastore.hpp @@ -23,7 +23,7 @@ DataStoreT<RefT>::~DataStoreT() = default; template <typename RefT> void -DataStoreT<RefT>::freeElem(EntryRef ref, size_t numElems) +DataStoreT<RefT>::free_elem_internal(EntryRef ref, size_t numElems, bool was_held) { RefType intRef(ref); BufferState &state = getBufferState(intRef.bufferId()); @@ -35,9 +35,12 @@ DataStoreT<RefT>::freeElem(EntryRef ref, size_t numElems) state.freeList().push_back(ref); } } else { - assert(state.isOnHold()); + assert(state.isOnHold() && was_held); } state.incDeadElems(numElems); + if (was_held) { + state.decHoldElems(numElems); + } state.cleanHold(getBuffer(intRef.bufferId()), intRef.unscaled_offset() * state.getArraySize(), numElems); } @@ -71,10 +74,7 @@ DataStoreT<RefT>::trimElemHoldList(generation_t usedGen) for (; it != ite; ++it) { if (static_cast<sgeneration_t>(it->_generation - usedGen) >= 0) break; - RefType intRef(it->_ref); - BufferState &state = getBufferState(intRef.bufferId()); - freeElem(it->_ref, it->_len); - state.decHoldElems(it->_len); + free_elem_internal(it->_ref, it->_len, true); ++freed; } if (freed != 0) { @@ -91,10 +91,7 @@ DataStoreT<RefT>::clearElemHoldList() ElemHold2List::iterator it(elemHold2List.begin()); ElemHold2List::iterator ite(elemHold2List.end()); for (; it != ite; ++it) { - RefType intRef(it->_ref); - BufferState &state = getBufferState(intRef.bufferId()); - freeElem(it->_ref, it->_len); - state.decHoldElems(it->_len); + free_elem_internal(it->_ref, it->_len, true); } elemHold2List.clear(); } diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index d6dcdee0eb4..a74e41dfb3f 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -449,7 +449,7 @@ DocsumFilter::getMappedDocsum(uint32_t id) _packer.Init(resClass->GetClassID()); for (FieldSpecList::iterator it(_fields.begin()), end = _fields.end(); it != end; ++it) { ResType type = it->getResultType(); - if (type == RES_JSONSTRING || type == RES_XMLSTRING) { + if (type == RES_JSONSTRING) { // this really means 'structured data' writeSlimeField(*it, doc, _packer); } else { |