diff options
43 files changed, 325 insertions, 234 deletions
diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index b465a3b8add..c94faa0d5e3 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -81,6 +81,6 @@ func printPublicKey(apiKeyFile, tenant string) { log.Printf("\nThis is your public key:\n%s", color.Green(pemPublicKey)) log.Printf("Its fingerprint is:\n%s\n", color.Cyan(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") - log.Printf(color.Cyan("https://console.vespa.oath.cloud/tenant/%s/keys").String(), tenant) + log.Printf(color.Cyan("%s/tenant/%s/keys").String(), defaultConsoleURL, tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index f44d3aeae42..19fa08ebaa4 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -61,13 +61,21 @@ If application directory is not specified, it defaults to working directory.`, opts.APIKey = readAPIKey(deployment.Application.Tenant) opts.Deployment = deployment } - if err := vespa.Deploy(opts); err == nil { - printSuccess("Deployed ", color.Cyan(pkg.Path)) + if sessionOrRunID, err := vespa.Deploy(opts); err == nil { if opts.IsCloud() { - log.Printf("\nUse %s for deployment status, or see", color.Cyan("vespa status")) - log.Print(color.Cyan(fmt.Sprintf("https://console.vespa.oath.cloud/tenant/%s/application/%s/dev/instance/%s", opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance))) + printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) + } else { + printSuccess("Deployed ", color.Cyan(pkg.Path)) } - waitForQueryService() + if opts.IsCloud() { + log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) + log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", + defaultConsoleURL, + opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance, + opts.Deployment.Zone.Environment, opts.Deployment.Zone.Region, + sessionOrRunID))) + } + waitForQueryService(sessionOrRunID) } else { fatalErr(nil, err.Error()) } @@ -123,17 +131,17 @@ var activateCmd = &cobra.Command{ }) if err == nil { printSuccess("Activated ", color.Cyan(pkg.Path), " with session ", sessionID) - waitForQueryService() + waitForQueryService(sessionID) } else { fatalErr(nil, err.Error()) } }, } -func waitForQueryService() { +func waitForQueryService(sessionOrRunID int64) { if waitSecsArg > 0 { log.Println() - waitForService("query") + waitForService("query", sessionOrRunID) } } diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index 8a438ad4125..450f061f140 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -107,7 +107,7 @@ var documentGetCmd = &cobra.Command{ }, } -func documentService() *vespa.Service { return getService("document") } +func documentService() *vespa.Service { return getService("document", 0) } func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) { if !result.Success { diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index 6e13c68f311..59af042e611 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -93,7 +93,7 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", executeCommand(t, client, arguments, []string{})) - target := getService("document").BaseURL + target := getService("document", 0).BaseURL expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) assert.Equal(t, target+"/document/v1/"+expectedPath, client.lastRequest.URL.String()) assert.Equal(t, "application/json", client.lastRequest.Header.Get("Content-Type")) @@ -115,7 +115,7 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { } `, executeCommand(t, client, arguments, []string{})) - target := getService("document").BaseURL + target := getService("document", 0).BaseURL expectedPath, _ := vespa.IdToURLPath(documentId) assert.Equal(t, target+"/document/v1/"+expectedPath, client.lastRequest.URL.String()) assert.Equal(t, "GET", client.lastRequest.Method) diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 67124ec2417..b672419cae6 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -17,6 +17,8 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) +const defaultConsoleURL = "https://console.vespa.oath.cloud" + var exitFunc = os.Exit // To allow overriding Exit in tests func fatalErrHint(err error, hints ...string) { @@ -94,13 +96,13 @@ func getTargetType() string { return target } -func getService(service string) *vespa.Service { +func getService(service string, sessionOrRunID int64) *vespa.Service { t := getTarget() timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { log.Printf("Waiting up to %d %s for services to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) } - if err := t.DiscoverServices(timeout); err != nil { + if err := t.DiscoverServices(timeout, sessionOrRunID); err != nil { fatalErr(err, "Services unavailable") } s, err := t.Service(service) @@ -134,8 +136,8 @@ func getTarget() vespa.Target { return nil } -func waitForService(service string) { - s := getService(service) +func waitForService(service string, sessionOrRunID int64) { + s := getService(service, sessionOrRunID) timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { log.Printf("Waiting up to %d %s for service to become ready ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index 062ea8d1bbc..ea80c037721 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -36,7 +36,7 @@ can be set by the syntax [parameter-name]=[value].`, } func query(arguments []string) { - service := getService("query") + service := getService("query", 0) url, _ := url.Parse(service.BaseURL + "/search/") urlQuery := url.Query() for i := 0; i < len(arguments); i++ { diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index 9bb5f07c3a1..af9f9c4cfd5 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -49,7 +49,7 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { "{\n \"query\": \"result\"\n}\n", executeCommand(t, client, []string{"query"}, query), "query output") - assert.Equal(t, getService("query").BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) + assert.Equal(t, getService("query", 0).BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } func assertQueryNonJsonResult(t *testing.T, expectedQuery string, query ...string) { @@ -58,7 +58,7 @@ func assertQueryNonJsonResult(t *testing.T, expectedQuery string, query ...strin "query result\n", executeCommand(t, client, []string{"query"}, query), "query output") - assert.Equal(t, getService("query").BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) + assert.Equal(t, getService("query", 0).BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } func assertQueryError(t *testing.T, status int, errorMessage string) { diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index 796ede9c86d..5fdcaa07d8a 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -22,7 +22,7 @@ var statusCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - waitForService("query") + waitForService("query", 0) }, } @@ -33,7 +33,7 @@ var statusQueryCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { - waitForService("query") + waitForService("query", 0) }, } @@ -44,7 +44,7 @@ var statusDocumentCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { - waitForService("document") + waitForService("document", 0) }, } @@ -55,6 +55,6 @@ var statusDeployCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { - waitForService("deploy") + waitForService("deploy", 0) }, } diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index c2a27442a53..081e9fc17d2 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -198,17 +198,17 @@ func Activate(sessionID int64, deployment DeploymentOpts) error { return checkResponse(req, response, serviceDescription) } -func Deploy(opts DeploymentOpts) error { +func Deploy(opts DeploymentOpts) (int64, error) { path := "/application/v2/tenant/default/prepareandactivate" if opts.IsCloud() { if !opts.ApplicationPackage.HasCertificate() { - return fmt.Errorf("%s: missing certificate in package", opts) + return 0, fmt.Errorf("%s: missing certificate in package", opts) } if opts.APIKey == nil { - return fmt.Errorf("%s: missing api key", opts.String()) + return 0, fmt.Errorf("%s: missing api key", opts.String()) } if opts.Deployment.Zone.Environment == "" || opts.Deployment.Zone.Region == "" { - return fmt.Errorf("%s: missing zone", opts) + return 0, fmt.Errorf("%s: missing zone", opts) } path = fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/deploy/%s-%s", opts.Deployment.Application.Tenant, @@ -219,10 +219,9 @@ func Deploy(opts DeploymentOpts) error { } u, err := opts.url(path) if err != nil { - return err + return 0, err } - _, err = uploadApplicationPackage(u, opts) - return err + return uploadApplicationPackage(u, opts) } func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) { @@ -251,16 +250,20 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) } defer response.Body.Close() - var sessionResponse struct { - SessionID string `json:"session-id"` + var jsonResponse struct { + SessionID string `json:"session-id"` // Config server + RunID int64 `json:"run"` // Controller } - sessionResponse.SessionID = "0" // Set a default session ID for responses that don't contain int (e.g. cloud deployment) + jsonResponse.SessionID = "0" // Set a default session ID for responses that don't contain int (e.g. cloud deployment) if err := checkResponse(request, response, serviceDescription); err != nil { return 0, err } jsonDec := json.NewDecoder(response.Body) - jsonDec.Decode(&sessionResponse) // Ignore error in case this is a non-JSON response - return strconv.ParseInt(sessionResponse.SessionID, 10, 64) + jsonDec.Decode(&jsonResponse) // Ignore error in case this is a non-JSON response + if jsonResponse.RunID > 0 { + return jsonResponse.RunID, nil + } + return strconv.ParseInt(jsonResponse.SessionID, 10, 64) } func checkResponse(req *http.Request, response *http.Response, serviceDescription string) error { diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 493f3ec3485..faf80736293 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -41,8 +41,9 @@ type Target interface { // Service returns the service for given name. Service(name string) (*Service, error) - // DiscoverServices queries for services available on this target until one is found or timeout passes. - DiscoverServices(timeout time.Duration) error + // DiscoverServices queries for services available on this target after the given session or deployment run has + // completed. + DiscoverServices(timeout time.Duration, sessionOrRunID int64) error } type customTarget struct { @@ -75,9 +76,8 @@ func (s *Service) Wait(timeout time.Duration) (int, error) { if err != nil { return 0, err } - okFunc := func(status int, response []byte) (string, bool) { return "", status/100 == 2 } - status, _, err := wait(okFunc, req, s.certificate, timeout) - return status, err + okFunc := func(status int, response []byte) (bool, error) { return status/100 == 2, nil } + return wait(okFunc, req, s.certificate, timeout) } func (s *Service) Description() string { @@ -103,7 +103,7 @@ func (t *customTarget) Service(name string) (*Service, error) { return nil, fmt.Errorf("unknown service: %s", name) } -func (t *customTarget) DiscoverServices(timeout time.Duration) error { return nil } +func (t *customTarget) DiscoverServices(timeout time.Duration, sessionID int64) error { return nil } func (t *localTarget) Type() string { return t.targetType } @@ -117,7 +117,7 @@ func (t *localTarget) Service(name string) (*Service, error) { return nil, fmt.Errorf("unknown service: %s", name) } -func (t *localTarget) DiscoverServices(timeout time.Duration) error { return nil } +func (t *localTarget) DiscoverServices(timeout time.Duration, sessionID int64) error { return nil } type cloudTarget struct { cloudAPI string @@ -150,39 +150,85 @@ func (t *cloudTarget) Service(name string) (*Service, error) { return nil, fmt.Errorf("unknown service: %s", name) } -// DiscoverServices queries Vespa Cloud for endpoints until at least one endpoint is returned, or timeout passes. -func (t *cloudTarget) DiscoverServices(timeout time.Duration) error { +// DiscoverServices waits for run identified by runID to complete and at least one endpoint is available, or timeout +// passes. +func (t *cloudTarget) DiscoverServices(timeout time.Duration, runID int64) error { + signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) + if runID > 0 { + if err := t.waitForRun(signer, runID, timeout); err != nil { + return err + } + } + return t.discoverEndpoints(signer, timeout) +} + +func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout time.Duration) error { + runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d", + t.cloudAPI, + t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, + t.deployment.Zone.Environment, t.deployment.Zone.Region, runID) + req, err := http.NewRequest("GET", runURL, nil) + if err != nil { + return err + } + if err := signer.SignRequest(req); err != nil { + return err + } + jobSuccessFunc := func(status int, response []byte) (bool, error) { + if status/100 != 2 { + return false, nil + } + var resp jobResponse + if err := json.Unmarshal(response, &resp); err != nil { + return false, nil + } + if resp.Active { + return false, nil + } + if resp.Status != "success" { + return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status) + } + return true, nil + } + _, err = wait(jobSuccessFunc, req, t.keyPair, timeout) + return err +} + +func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Duration) error { deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s", t.cloudAPI, t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, t.deployment.Zone.Environment, t.deployment.Zone.Region) req, err := http.NewRequest("GET", deploymentURL, nil) - signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) + if err != nil { + return err + } if err := signer.SignRequest(req); err != nil { return err } - endpointFunc := func(status int, response []byte) (string, bool) { + var endpointURL string + endpointFunc := func(status int, response []byte) (bool, error) { if status/100 != 2 { - return "", false + return false, nil } var resp deploymentResponse if err := json.Unmarshal(response, &resp); err != nil { - return "", false + return false, nil } if len(resp.Endpoints) == 0 { - return "", false + return false, nil } - return resp.Endpoints[0].URL, true + endpointURL = resp.Endpoints[0].URL + return true, nil } - _, endpoint, err := wait(endpointFunc, req, t.keyPair, timeout) - if err != nil { + if _, err = wait(endpointFunc, req, t.keyPair, timeout); err != nil { return err } - if endpoint == "" { + if endpointURL == "" { return fmt.Errorf("no endpoint discovered") } - t.queryURL = endpoint - t.documentURL = endpoint + t.queryURL = endpointURL + t.documentURL = endpointURL return nil } @@ -213,9 +259,14 @@ type deploymentResponse struct { Endpoints []deploymentEndpoint `json:"endpoints"` } -type responseFunc func(status int, response []byte) (string, bool) +type jobResponse struct { + Active bool `json:"active"` + Status string `json:"status"` +} -func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeout time.Duration) (int, string, error) { +type responseFunc func(status int, response []byte) (bool, error) + +func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeout time.Duration) (int, error) { if certificate.Certificate != nil { util.ActiveHttpClient.UseCertificate(certificate) } @@ -232,12 +283,15 @@ func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeo statusCode = response.StatusCode body, err := ioutil.ReadAll(response.Body) if err != nil { - return 0, "", err + return 0, err } response.Body.Close() - result, ok := fn(statusCode, body) + ok, err := fn(statusCode, body) + if err != nil { + return statusCode, err + } if ok { - return statusCode, result, nil + return statusCode, nil } } if loopOnce { @@ -245,5 +299,5 @@ func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeo } time.Sleep(waitRetryInterval) } - return statusCode, "", httpErr + return statusCode, httpErr } diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index bd9a93d9d24..1f46cc83178 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -23,6 +23,14 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request response = fmt.Sprintf(`{"endpoints": [{"url": "%s"}]}`, v.serverURL) } w.Write([]byte(response)) + case "/application/v4/tenant/t1/application/a1/instance/i1/job/dev-us-north-1/run/42": + response := "{}" + if v.endpointsReady { + response = `{"active": false, "status": "success"}` + } else { + response = `{"active": true, "status": "running"}` + } + w.Write([]byte(response)) case "/status.html": w.Write([]byte("OK")) case "/ApplicationStatus": @@ -76,11 +84,11 @@ func TestCloudTargetWait(t *testing.T) { _, err = target.Service("query") assert.NotNil(t, err) - err = target.DiscoverServices(0) + err = target.DiscoverServices(0, 42) assert.NotNil(t, err) vc.endpointsReady = true - err = target.DiscoverServices(0) + err = target.DiscoverServices(0, 42) assert.Nil(t, err) assertServiceWait(t, 500, target, "query") diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a1de4ff26d6..a6b56640c97 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -86,7 +86,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean distributeExternalRankExpressions() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean dryRunOnnxOnSetup() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean dryRunOnnxOnSetup() { return true; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 34d5533364f..b42686406b5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -2,7 +2,6 @@ package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; -import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -60,7 +59,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxMergeQueueSize = 1024; private int largeRankExpressionLimit = 8192; private boolean allowDisableMtls = true; - private boolean dryRunOnnxOnSetup = false; private List<X509Certificate> operatorCertificates = Collections.emptyList(); private double resourceLimitDisk = 0.8; private double resourceLimitMemory = 0.8; @@ -105,17 +103,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } - @Override public boolean dryRunOnnxOnSetup() { return dryRunOnnxOnSetup; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @Override public int metricsproxyNumThreads() { return 1; } @Override public boolean enforceRankProfileInheritance() { return true; } - public TestProperties setDryRunOnnxOnSetup(boolean value) { - dryRunOnnxOnSetup = value; - return this; - } public TestProperties useExternalRankExpression(boolean value) { useExternalRankExpression = value; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 6528c9195d4..122c39f440f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -40,7 +40,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private final RankingConstants rankingConstants; private final LargeRankExpressions largeRankExpressions; private final OnnxModels onnxModels; - private final boolean dryRunOnnxOnSetup; public static RankProfileList empty = new RankProfileList(); @@ -48,7 +47,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ rankingConstants = new RankingConstants(null); largeRankExpressions = new LargeRankExpressions(null); onnxModels = new OnnxModels(null); - dryRunOnnxOnSetup = true; } /** @@ -71,7 +69,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ this.rankingConstants = rankingConstants; this.largeRankExpressions = largeRankExpressions; this.onnxModels = onnxModels; // as ONNX models come from parsing rank expressions - dryRunOnnxOnSetup = deployProperties.featureFlags().dryRunOnnxOnSetup(); deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties, executor); } @@ -154,7 +151,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way else { OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); - modelBuilder.dry_run_on_setup(dryRunOnnxOnSetup); + modelBuilder.dry_run_on_setup(true); modelBuilder.name(model.getName()); modelBuilder.fileref(model.getFileReference()); model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java index ab148130a7d..f1f6860c197 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java @@ -2,10 +2,8 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -17,6 +15,7 @@ import org.junit.After; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class RankingExpressionWithOnnxModelTestCase { @@ -29,9 +28,9 @@ public class RankingExpressionWithOnnxModelTestCase { @Test public void testOnnxModelFeature() throws Exception { - VespaModel model = loadModel(applicationDir, false); + VespaModel model = loadModel(applicationDir); assertTransformedFeature(model); - assertGeneratedConfig(model, false); + assertGeneratedConfig(model); Path storedApplicationDir = applicationDir.append("copy"); try { @@ -41,30 +40,29 @@ public class RankingExpressionWithOnnxModelTestCase { IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedApplicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - VespaModel storedModel = loadModel(storedApplicationDir, true); + VespaModel storedModel = loadModel(storedApplicationDir); assertTransformedFeature(storedModel); - assertGeneratedConfig(storedModel, true); + assertGeneratedConfig(storedModel); } finally { IOUtils.recursiveDeleteDir(storedApplicationDir.toFile()); } } - private VespaModel loadModel(Path path, boolean dryRunOnnx) throws Exception { + private VespaModel loadModel(Path path) throws Exception { FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); - ModelContext.Properties properties = new TestProperties().setDryRunOnnxOnSetup(dryRunOnnx); - DeployState state = new DeployState.Builder().applicationPackage(applicationPackage).properties(properties).build(); + DeployState state = new DeployState.Builder().applicationPackage(applicationPackage).build(); return new VespaModel(state); } - private void assertGeneratedConfig(VespaModel vespaModel, boolean expectDryRunOnnx) { + private void assertGeneratedConfig(VespaModel vespaModel) { DocumentDatabase db = ((IndexedSearchCluster)vespaModel.getSearchClusters().get(0)).getDocumentDbs().get(0); OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); ((OnnxModelsConfig.Producer) db).getConfig(builder); OnnxModelsConfig config = new OnnxModelsConfig(builder); assertEquals(6, config.model().size()); for (OnnxModelsConfig.Model model : config.model()) { - assertEquals(expectDryRunOnnx, model.dry_run_on_setup()); + assertTrue(model.dry_run_on_setup()); } OnnxModelsConfig.Model model = config.model(0); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index 317606ac3ef..f77bd4b9138 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -1,13 +1,13 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import java.util.logging.Level; -import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; -import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.yolean.Exceptions; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -44,7 +44,7 @@ public class DelayedResponseHandler implements Runnable { AtomicInteger i = new AtomicInteger(0); while ((response = delayedResponses.responses().poll()) != null) { JRTServerConfigRequest request = response.getRequest(); - ConfigCacheKey cacheKey = new ConfigCacheKey(request.getConfigKey(), request.getConfigKey().getMd5()); + ConfigCacheKey cacheKey = new ConfigCacheKey(request.getConfigKey(), request.getRequestDefMd5()); RawConfig config = memoryCache.get(cacheKey); if (config != null) { rpcServer.returnOkResponse(request, config); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java index 33e798da15e..1b009b80fc1 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; import com.yahoo.jrt.Request; @@ -25,6 +25,7 @@ import java.util.Optional; public class ConfigTester { private static final long defaultTimeout = 10000; + private static final List<String> defContent = Collections.singletonList("bar string"); static RawConfig fooConfig; static RawConfig barConfig; @@ -39,7 +40,6 @@ public class ConfigTester { ConfigPayload fooConfigPayload = createConfigPayload("bar", "value"); fooPayload = Payload.from(fooConfigPayload); - List<String> defContent = Collections.singletonList("bar string"); long generation = 1; String defMd5 = ConfigUtils.getDefMd5(defContent); String configMd5 = ConfigUtils.getMd5(fooConfigPayload); @@ -57,17 +57,28 @@ public class ConfigTester { JRTServerConfigRequest createRequest(RawConfig config, long timeout) { return createRequest(config.getName(), config.getConfigId(), config.getNamespace(), - config.getConfigMd5(), config.getDefMd5(), config.getGeneration(), timeout); + config.getConfigMd5(), config.getGeneration(), timeout); } JRTServerConfigRequest createRequest(String configName, String configId, String namespace, long timeout) { - return createRequest(configName, configId, namespace, "", null, 0, timeout); + return createRequest(configName, configId, namespace, null, 0, timeout); } - private JRTServerConfigRequest createRequest(String configName, String configId, String namespace, String md5, String defMd5, long generation, long timeout) { + private JRTServerConfigRequest createRequest(String configName, + String configId, + String namespace, + String md5, + long generation, + long timeout) { Request request = JRTClientConfigRequestV3. - createWithParams(new ConfigKey<>(configName, configId, namespace, defMd5, null), DefContent.fromList(Collections.emptyList()), - "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED, + createWithParams(new ConfigKey<>(configName, configId, namespace, null), + DefContent.fromList(defContent), + "fromHost", + md5, + generation, + timeout, + Trace.createDummy(), + CompressionType.UNCOMPRESSED, Optional.empty()).getRequest(); return JRTServerConfigRequestV3.createFromRequest(request); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java b/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java index 75e75e7b1f5..93b85aaabd0 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/PayloadChecksum.java @@ -12,7 +12,7 @@ import java.util.regex.Pattern; */ public class PayloadChecksum { - private static final Pattern hexChecksumPattern = Pattern.compile("[0-9a-zA-Z]+"); + private static final Pattern hexChecksumPattern = Pattern.compile("[0-9a-fA-F]+"); private final String checksum; private final Type type; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java index 6fcdc19cf97..9a7ac58e339 100755 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import com.yahoo.config.ConfigInstance; @@ -17,7 +17,6 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable // The two fields below are only set when ConfigKey is constructed from a config class. Can be null private final Class<CONFIGCLASS> configClass; - private final String md5; // config definition md5 /** * Constructs new key @@ -27,7 +26,7 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable * @param namespace namespace for this config definition */ public ConfigKey(String name, String configIdString, String namespace) { - this(name, configIdString, namespace, null, null); + this(name, configIdString, namespace, null); } /** @@ -38,10 +37,12 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable */ public ConfigKey(Class<CONFIGCLASS> clazz, String configIdString) { this(getFieldFromClass(clazz, "CONFIG_DEF_NAME"), - configIdString, getFieldFromClass(clazz, "CONFIG_DEF_NAMESPACE"), getFieldFromClass(clazz, "CONFIG_DEF_MD5"), clazz); + configIdString, + getFieldFromClass(clazz, "CONFIG_DEF_NAMESPACE"), + clazz); } - public ConfigKey(String name, String configIdString, String namespace, String defMd5, Class<CONFIGCLASS> clazz) { + public ConfigKey(String name, String configIdString, String namespace, Class<CONFIGCLASS> clazz) { if (name == null || name.isEmpty()) throw new ConfigurationRuntimeException("Config name cannot be null or empty!"); if (namespace == null || namespace.isEmpty()) @@ -49,7 +50,6 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable this.name = name; this.configId = (configIdString == null) ? "" : configIdString; this.namespace = namespace; - this.md5 = (defMd5 == null) ? "" : defMd5; this.configClass = clazz; } @@ -108,10 +108,6 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable return configClass; } - public String getMd5() { - return md5; - } - public String toString() { StringBuilder sb = new StringBuilder(); sb.append("name="); @@ -123,8 +119,8 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable return sb.toString(); } - public static ConfigKey<?> createFull(String name, String configId, String namespace, String md5) { - return new ConfigKey<>(name, configId, namespace, md5, null); + public static ConfigKey<?> createFull(String name, String configId, String namespace) { + return new ConfigKey<>(name, configId, namespace, null); } } diff --git a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java index c5eeab74157..4e90ce532e4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import com.yahoo.vespa.config.protocol.DefContent; @@ -37,4 +37,11 @@ public interface GetConfigRequest { */ boolean noCache(); + /** + * Returns the md5 of the config definition in the request. + * + * @return an md5 of config definition in request. + */ + String getRequestDefMd5(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java index 03e08e039ae..2a992721c86 100644 --- a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java +++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java @@ -1,9 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; -import com.yahoo.vespa.config.util.ConfigUtils; import java.nio.ByteBuffer; @@ -14,18 +13,12 @@ import java.nio.ByteBuffer; */ public class LZ4PayloadCompressor { - private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL"; - private static final Compressor compressor = new Compressor(CompressionType.LZ4, getCompressionLevel()); - - private static int getCompressionLevel() { - return Integer.parseInt(ConfigUtils.getEnvValue("0", - System.getenv(VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL), - System.getProperty(VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL))); - } + private static final Compressor compressor = new Compressor(CompressionType.LZ4, 0); public byte[] compress(byte[] input) { return compressor.compressUnconditionally(input); } + public byte[] compress(ByteBuffer input) { return compressor.compressUnconditionally(input); } @@ -33,6 +26,7 @@ public class LZ4PayloadCompressor { public byte [] decompress(byte[] input, int uncompressedLen) { return compressor.decompressUnconditionally(input, 0, uncompressedLen); } + public byte [] decompress(ByteBuffer input, int uncompressedLen) { ByteBuffer uncompressed = ByteBuffer.allocate(uncompressedLen); compressor.decompressUnconditionally(input, uncompressed); diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java index 366387f46af..028e84e4c29 100755 --- a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -77,7 +77,7 @@ public class RawConfig extends ConfigInstance { */ public static RawConfig createFromResponseParameters(JRTClientConfigRequest req) { return new RawConfig(req.getConfigKey(), - req.getConfigKey().getMd5(), + ConfigUtils.getDefMd5(req.getDefContent().asList()), req.getNewPayload(), req.getNewConfigMd5(), req.getNewGeneration(), @@ -94,7 +94,7 @@ public class RawConfig extends ConfigInstance { */ public static RawConfig createFromServerRequest(JRTServerConfigRequest req) { return new RawConfig(req.getConfigKey(), - req.getConfigKey().getMd5() , + ConfigUtils.getDefMd5(req.getDefContent().asList()), Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), req.getRequestConfigMd5(), req.getRequestGeneration(), diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java index 26c5889c579..adb27f37413 100644 --- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.benchmark; import com.yahoo.collections.Tuple2; @@ -252,7 +252,7 @@ public class LoadTester { private JRTClientConfigRequest createRequest(ConfigKey<?> reqKey) { ConfigDefinitionKey dKey = new ConfigDefinitionKey(reqKey); Tuple2<String, String[]> defContent = defs.get(dKey); - ConfigKey<?> fullKey = createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace(), defContent.first); + ConfigKey<?> fullKey = createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace()); final long serverTimeout = 1000; return JRTClientConfigRequestV3.createWithParams(fullKey, DefContent.fromList(List.of(defContent.second)), diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java index 23e3259ff8b..f5b558550e4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -260,6 +260,11 @@ public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { } @Override + public String getRequestDefMd5() { + return requestData.getRequestDefMd5(); + } + + @Override public boolean validateResponse() { if (request.isError()) { return false; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java index 0e88eeb4e98..5b8f040b8e3 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; import com.yahoo.jrt.Request; @@ -14,14 +14,14 @@ import java.util.Optional; public interface JRTConfigRequest { /** - * Get the config key of the config request. + * Returns the config key of the config request. * * @return a {@link ConfigKey}. */ ConfigKey<?> getConfigKey(); /** - * Perform request parameter validation of this config request. This method should be called before fetching + * Performs request parameter validation of this config request. This method should be called before fetching * any kind of config protocol-specific parameter. * * @return true if valid, false if not. @@ -29,21 +29,28 @@ public interface JRTConfigRequest { boolean validateParameters(); /** - * Get the config md5 of the config request. Return an empty string if no response has been returned. + * Returns the config md5 of the config request. Return an empty string if no response has been returned. * * @return a config md5. */ String getRequestConfigMd5(); /** - * Get the generation of the requested config. If none has been given, 0 should be returned. + * Returns the md5 of the config definition in the request. + * + * @return an md5 of config definition in request. + */ + String getRequestDefMd5(); + + /** + * Returns the generation of the requested config. If none has been given, 0 should be returned. * * @return the generation in the request. */ long getRequestGeneration(); /** - * Get the JRT request object for this config request. + * Returns the JRT request object for this config request. * TODO: This method leaks the internal jrt stuff :( * * @return a {@link Request} object. @@ -51,14 +58,14 @@ public interface JRTConfigRequest { Request getRequest(); /** - * Get a short hand description of this request. + * Returns a short hand description of this request. * * @return a short description */ String getShortDescription(); /** - * Get the error code of this request + * Returns the error code of this request * * @return the error code as defined in {@link com.yahoo.vespa.config.ErrorCode}. */ @@ -72,27 +79,27 @@ public interface JRTConfigRequest { String errorMessage(); /** - * Get the server timeout of this request. + * Returns the server timeout of this request. * * @return the timeout given to the server */ long getTimeout(); /** - * Get the config protocol version + * Returns the config protocol version * * @return a protocol version number. */ long getProtocolVersion(); /** - * Get the host name of the client that is requesting config. + * Returns the host name of the client that is requesting config. * @return hostname of the client. */ String getClientHostName(); /** - * Get the Vespa version of the client that initiated the request + * Returns the Vespa version of the client that initiated the request * * @return Vespa version of the client */ diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java index c085be5924c..abc2b0b4473 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -46,6 +46,13 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque String getRequestConfigMd5(); /** + * Returns the md5 of the config definition in the request. + * + * @return an md5 of config definition in request. + */ + String getRequestDefMd5(); + + /** * Get the current config generation of the client config. * * @return the current config generation. diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index 4575368cc6a..e0a5b23a6d4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; import com.fasterxml.jackson.core.JsonFactory; @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.util.ConfigUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.Optional; import java.util.logging.Logger; @@ -192,6 +191,9 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { return requestData.getRequestConfigMd5(); } + @Override + public String getRequestDefMd5() { return requestData.getRequestDefMd5(); } + private void addErrorResponse(int errorCode) { addErrorResponse(errorCode, ErrorCode.getName(errorCode)); } @@ -222,7 +224,7 @@ public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, requestData.getRequestDefMd5()); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 81ea99e38b4..7db15844e8b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -29,7 +29,7 @@ public class RequestValidation { log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } - if (!(new PayloadChecksum(key.getMd5()).valid())) { + if (!(new PayloadChecksum(request.getRequestDefMd5()).valid())) { log.log(INFO, "Illegal checksum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; // TODO: Use ILLEGAL_DEF_CHECKSUM } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java index 447bc63eeb6..b885623a78b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; import com.yahoo.jrt.Request; @@ -7,6 +7,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.util.ConfigUtils; import java.util.Optional; @@ -57,9 +58,8 @@ class SlimeRequestData { ConfigKey<?> getConfigKey() { return ConfigKey.createFull(getRequestField(REQUEST_DEF_NAME).asString(), - getRequestField(REQUEST_CLIENT_CONFIGID).asString(), - getRequestField(REQUEST_DEF_NAMESPACE).asString(), - getRequestField(REQUEST_DEF_MD5).asString()); + getRequestField(REQUEST_CLIENT_CONFIGID).asString(), + getRequestField(REQUEST_DEF_NAMESPACE).asString()); } DefContent getSchema() { @@ -75,9 +75,9 @@ class SlimeRequestData { return getRequestField(REQUEST_TIMEOUT).asLong(); } - String getRequestConfigMd5() { - return getRequestField(REQUEST_CONFIG_MD5).asString(); - } + String getRequestConfigMd5() { return getRequestField(REQUEST_CONFIG_MD5).asString(); } + + String getRequestDefMd5() { return getRequestField(REQUEST_DEF_MD5).asString(); } long getRequestGeneration() { return getRequestField(REQUEST_CURRENT_GENERATION).asLong(); @@ -98,7 +98,7 @@ class SlimeRequestData { request.setLong(REQUEST_VERSION, protocolVersion); request.setString(REQUEST_DEF_NAME, key.getName()); request.setString(REQUEST_DEF_NAMESPACE, key.getNamespace()); - request.setString(REQUEST_DEF_MD5, key.getMd5()); + request.setString(REQUEST_DEF_MD5, ConfigUtils.getDefMd5(defSchema.asList())); request.setString(REQUEST_CLIENT_CONFIGID, key.getConfigId()); request.setString(REQUEST_CLIENT_HOSTNAME, hostname); defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT)); diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java index 452d0d78897..e6bc13248fd 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.foo.AppConfig; +import org.junit.Test; + import java.util.ArrayList; import java.util.Collections; import java.util.List; -import com.yahoo.foo.AppConfig; -import com.yahoo.config.ConfigurationRuntimeException; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -52,7 +52,6 @@ public class ConfigKeyTest { assertEquals("Name is set correctly from class", name, classKey.getName()); assertEquals("Namespace is set correctly from class", namespace, classKey.getNamespace()); assertEquals(configId, classKey.getConfigId()); - assertEquals("Md5 is set correctly from class", md5, classKey.getMd5()); ConfigKey<?> stringKey = new ConfigKey<>(name, configId, namespace); assertEquals("Key created from class equals key created from strings", stringKey, classKey); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index b67a9ef0fb8..d6ce246aa1f 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -23,8 +23,8 @@ import com.yahoo.vespa.config.util.ConfigUtils; import org.junit.Before; import org.junit.Test; -import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; @@ -39,12 +39,17 @@ import static org.junit.Assert.assertTrue; */ public class JRTConfigRequestV3Test { + private static final String [] configDefinition = new String[]{ + "namespace=my.name.space", + "myfield string" + }; + private final Optional<VespaVersion> vespaVersion = Optional.of(VespaVersion.fromString("5.38.24")); private final String defName = "mydef"; private final String defNamespace = "my.name.space"; private final String hostname = "myhost"; private final String configId = "config/id"; - private final String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1"; + private final String defMd5 = ConfigUtils.getDefMd5(List.of(configDefinition)); private final long currentGeneration = 3; private final long timeout = 5000; private Trace trace ; @@ -220,13 +225,12 @@ public class JRTConfigRequestV3Test { @Test public void parameters_are_validated() { assertTrue(serverReq.validateParameters()); - assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, "34", hostname, configId, "34", currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, "34", currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, -34, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, -23, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, "", configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq("35#$#!$@#", defNamespace, hostname, configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, "abcd.o#$*(!&$", hostname, configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, hostname, configId, "34", currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, hostname, configId, configMd5, -34, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, hostname, configId, configMd5, currentGeneration, -23, trace)); + assertValidationFail(createReq(defName, defNamespace, "", configId, configMd5, currentGeneration, timeout, trace)); } private void assertValidationFail(JRTClientConfigRequest req) { @@ -243,11 +247,11 @@ public class JRTConfigRequestV3Test { return Payload.from(new ConfigPayload(slime)); } - private JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5, + private JRTClientConfigRequest createReq(String defName, String defNamespace, String hostname, String configId, String configMd5, long currentGeneration, long timeout, Trace trace) { - return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace, defMd5), - DefContent.fromList(Arrays.asList("namespace=my.name.space", "myfield string")), + return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace), + DefContent.fromList(List.of(configDefinition)), hostname, configMd5, currentGeneration, @@ -272,26 +276,22 @@ public class JRTConfigRequestV3Test { private JRTClientConfigRequest createReq() { trace = Trace.createNew(3, new ManualClock()); trace.trace(1, "hei"); - return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace); + return createReq(defName, defNamespace, hostname, configId, configMd5, currentGeneration, timeout, trace); } private JRTClientConfigRequest createReq(Payload payload) { trace = Trace.createNew(3, new ManualClock()); trace.trace(1, "hei"); - return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); + return createReq(defName, defNamespace, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); } private void request_is_parsed_base() { - String [] expectedContent = new String[]{ - "namespace=my.name.space", - "myfield string" - }; System.out.println(serverReq.toString()); assertThat(serverReq.getConfigKey().getName(), is(defName)); assertThat(serverReq.getConfigKey().getNamespace(), is(defNamespace)); - assertThat(serverReq.getConfigKey().getMd5(), is(defMd5)); + assertThat(serverReq.getRequestDefMd5(), is(defMd5)); assertThat(serverReq.getConfigKey().getConfigId(), is(configId)); - assertThat(serverReq.getDefContent().asStringArray(), is(expectedContent)); + assertThat(serverReq.getDefContent().asStringArray(), is(configDefinition)); assertFalse(serverReq.noCache()); assertTrue(serverReq.getRequestTrace().toString().contains("hi")); assertThat(serverReq.getRequestConfigMd5(), is(configMd5)); @@ -299,16 +299,12 @@ public class JRTConfigRequestV3Test { } private JRTServerConfigRequest next_request_is_correct_base() { - String [] expectedContent = new String[]{ - "namespace=my.name.space", - "myfield string" - }; JRTServerConfigRequest next = createReq(clientReq.nextRequest(6).getRequest()); assertThat(next.getConfigKey().getName(), is(defName)); assertThat(next.getConfigKey().getNamespace(), is(defNamespace)); - assertThat(next.getConfigKey().getMd5(), is(defMd5)); + assertThat(next.getRequestDefMd5(), is(defMd5)); assertThat(next.getConfigKey().getConfigId(), is(configId)); - assertThat(next.getDefContent().asStringArray(), is(expectedContent)); + assertThat(next.getDefContent().asStringArray(), is(configDefinition)); assertFalse(next.noCache()); assertThat(next.getTimeout(), is(6L)); assertThat(next.getTimeout(), is(6L)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 0afb3049f6e..df1427bdf6d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; import com.yahoo.collections.Pair; @@ -102,7 +102,7 @@ public class Application implements ModelResult { long start = System.currentTimeMillis(); metricUpdater.incrementRequests(); ConfigKey<?> configKey = req.getConfigKey(); - String defMd5 = configKey.getMd5(); + String defMd5 = req.getRequestDefMd5(); if (defMd5 == null || defMd5.isEmpty()) { defMd5 = ConfigUtils.getDefMd5(req.getDefContent().asList()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 9a736fc6d01..414782a43f4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -179,7 +179,6 @@ public class ModelContextImpl implements ModelContext { private final int maxMergeQueueSize; private final int largeRankExpressionLimit; private final boolean throwIfResourceLimitsSpecified; - private final boolean dryRunOnnxOnSetup; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; @@ -210,7 +209,6 @@ public class ModelContextImpl implements ModelContext { this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); this.throwIfResourceLimitsSpecified = flagValue(source, appId, Flags.THROW_EXCEPTION_IF_RESOURCE_LIMITS_SPECIFIED); - this.dryRunOnnxOnSetup = flagValue(source, appId, Flags.DRY_RUN_ONNX_ON_SETUP); this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); @@ -243,7 +241,6 @@ public class ModelContextImpl implements ModelContext { @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean throwIfResourceLimitsSpecified() { return throwIfResourceLimitsSpecified; } - @Override public boolean dryRunOnnxOnSetup() { return dryRunOnnxOnSetup; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java index cdbd7378151..c01008fafa0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -1,13 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import java.util.Collections; -import java.util.Optional; -import java.util.Set; - import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.yahoo.collections.Tuple2; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -17,11 +14,14 @@ import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; import com.yahoo.vespa.config.server.http.v2.request.TenantRequest; import com.yahoo.vespa.config.util.ConfigUtils; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + /** * A request to get config, bound to tenant and app id. Used by both v1 and v2 of the config REST API. * @@ -191,4 +191,8 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { public boolean noCache() { return noCache; } + + @Override + public String getRequestDefMd5() { return ConfigUtils.getDefMd5(getDefContent().asList()); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java index 37c91cf255d..e4e19743950 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; import com.google.inject.Inject; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d81bf6fada3..41af0296534 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; @@ -39,7 +39,6 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; -import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; @@ -807,6 +806,10 @@ public class ApplicationRepositoryTest { public boolean noCache() { return false; } + + @Override + public String getRequestDefMd5() { return ""; } + }, Optional.empty()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index c58bd6d6b0a..eb1e541a540 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -74,7 +74,7 @@ public class SuperModelControllerTest { @Test(expected = UnknownConfigDefinitionException.class) public void test_unknown_config_definition() { String md5 = "asdfasf"; - Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", md5, null), DefContent.fromList(Collections.emptyList()), + Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", null), DefContent.fromList(Collections.emptyList()), "fromHost", md5, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()) .getRequest(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java index acd0acc3792..44491667760 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java @@ -105,7 +105,7 @@ public class ApplicationTest { @Test(expected = UnknownConfigDefinitionException.class) public void require_that_def_file_must_exist() { - handler.resolveConfig(createRequest("unknown", "namespace", "a", emptySchema)); + handler.resolveConfig(createRequest("unknown", "namespace", emptySchema)); } @Test @@ -116,8 +116,9 @@ public class ApplicationTest { @Test public void require_that_build_config_can_be_resolved() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, - ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)) + handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, + ModelConfig.CONFIG_DEF_NAMESPACE, + ModelConfig.CONFIG_DEF_SCHEMA)) .serialize(baos, CompressionType.UNCOMPRESSED); assertTrue(baos.toString().startsWith("{\"vespaVersion\":\"1.0.0\",\"hosts\":[{\"name\":\"mytesthost\"")); } @@ -132,23 +133,23 @@ public class ApplicationTest { String[] schema = new String[1]; schema[0] = "boolval bool default=true"; // changed to be true, original is false baos = new ByteArrayOutputStream(); - handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)) + handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, schema)) .serialize(baos, CompressionType.UNCOMPRESSED); assertEquals("{\"boolval\":true,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(Utf8.getCharset())); } @Test public void require_that_configs_are_cached() { - ConfigResponse response = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)); + ConfigResponse response = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_SCHEMA)); assertNotNull(response); - ConfigResponse cached_response = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)); + ConfigResponse cached_response = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_SCHEMA)); assertNotNull(cached_response); assertTrue(response == cached_response); } - private static GetConfigRequest createRequest(String name, String namespace, String defMd5, String[] schema) { + private static GetConfigRequest createRequest(String name, String namespace, String[] schema) { Request request = JRTClientConfigRequestV3. - createWithParams(new ConfigKey<>(name, "admin/model", namespace, defMd5, null), DefContent.fromArray(schema), + createWithParams(new ConfigKey<>(name, "admin/model", namespace, null), DefContent.fromArray(schema), "fromHost", "", 0, 100, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()).getRequest(); return JRTServerConfigRequestV3.createFromRequest(request); @@ -157,7 +158,7 @@ public class ApplicationTest { private static GetConfigRequest createSimpleConfigRequest() { return createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, - SimpletypesConfig.CONFIG_DEF_MD5, ApplicationTest.emptySchema); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java index 8c62fe77557..738e8c9827d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; import com.yahoo.config.provision.ApplicationId; @@ -29,7 +29,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class DelayedConfigResponseTest { @@ -42,7 +41,7 @@ public class DelayedConfigResponseTest { MockRpcServer rpc = new MockRpcServer(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); assertThat(responses.size(), is(0)); - JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 1000000, "bar"); + JRTServerConfigRequest req = createRequest("foo", "myid", "mymd5", 3, 1000000, "bar"); req.setDelayedResponse(true); GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId()); responses.delayResponse(req, context); @@ -50,7 +49,7 @@ public class DelayedConfigResponseTest { req.setDelayedResponse(false); responses.delayResponse(req, context); - responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); + responses.delayResponse(createRequest("foolio", "myid", "mymd5", 3, 100000, "bar"), context); assertThat(responses.size(), is(2)); assertTrue(req.isDelayedResponse()); List<DelayedConfigResponses.DelayedConfigResponse> it = responses.allDelayedResponses(); @@ -62,7 +61,7 @@ public class DelayedConfigResponseTest { GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId()); MockRpcServer rpc = new MockRpcServer(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); - responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); + responses.delayResponse(createRequest("foolio", "myid", "mymd5", 3, 100000, "bar"), context); assertThat(responses.size(), is(1)); responses.allDelayedResponses().get(0).cancelAndRemove(); assertThat(responses.size(), is(0)); @@ -74,15 +73,15 @@ public class DelayedConfigResponseTest { DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); assertThat(responses.size(), is(0)); assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0")); - JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 100, "bar"); + JRTServerConfigRequest req = createRequest("foo", "myid", "mymd5", 3, 100, "bar"); responses.delayResponse(req, GetConfigContext.testContext(ApplicationId.defaultId())); rpc.waitUntilSet(Duration.ofSeconds(5)); assertThat(rpc.latestRequest, is(req)); } - private JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) { + private JRTServerConfigRequest createRequest(String configName, String configId, String md5, long generation, long timeout, String namespace) { Request request = JRTClientConfigRequestV3. - createWithParams(new ConfigKey<>(configName, configId, namespace, defMd5, null), DefContent.fromList(Collections.emptyList()), + createWithParams(new ConfigKey<>(configName, configId, namespace, null), DefContent.fromList(Collections.emptyList()), "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()).getRequest(); return JRTServerConfigRequestV3.createFromRequest(request); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java index 12debc347de..8d402f57103 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java @@ -264,7 +264,7 @@ public class MultiTenantRpcAuthorizerTest { Cursor request = data.setObject(); request.setString("defName", configKey.getName()); request.setString("defNamespace", configKey.getNamespace()); - request.setString("defMD5", configKey.getMd5()); + request.setString("defMD5", ""); request.setString("configId", configKey.getConfigId()); request.setString("clientHostname", hostname); try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 0f5c2123325..52c192cfeab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -1,6 +1,7 @@ // Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.archive; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; @@ -45,7 +46,8 @@ public class CuratorArchiveBucketDb { public CuratorArchiveBucketDb(Controller controller) { this.archiveService = controller.serviceRegistry().archiveService(); this.curatorDb = controller.curator(); - this.enabled = controller.zoneRegistry().system().isPublic(); + SystemName system = controller.zoneRegistry().system(); + this.enabled = system.isPublic() || system.isCd(); } public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant) { diff --git a/dist/vespa.spec b/dist/vespa.spec index 6e4db510929..a66fc08038e 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -82,7 +82,8 @@ BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-icu-devel >= 65.1.0-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 -BuildRequires: vespa-openssl-devel >= 1.1.1k-1 +BuildRequires: vespa-openssl-devel >= 1.1.1l-1 +%define _use_vespa_openssl 1 BuildRequires: vespa-protobuf-devel = 3.17.3 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif @@ -104,7 +105,8 @@ BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12) BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11) %endif BuildRequires: vespa-boost-devel >= 1.76.0-1 -BuildRequires: openssl-devel +BuildRequires: vespa-openssl-devel >= 1.1.1l-1 +%define _use_vespa_openssl 1 BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.7.1 @@ -283,7 +285,7 @@ Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private # _dl_sym function. # Exclude automated requires for libraries in /opt/vespa-deps/lib64. -%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(crypto|icui18n|icuuc|lz4|protobuf|ssl|zstd|onnxruntime%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ +%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ %description @@ -319,8 +321,8 @@ Requires: vespa-xxhash = 0.8.0 %else Requires: xxhash-libs >= 0.8.0 %endif -%if 0%{?el7} -Requires: vespa-openssl >= 1.1.1k-1 +%if 0%{?el7} || 0%{?el8} +Requires: vespa-openssl >= 1.1.1l-1 %else Requires: openssl-libs %endif @@ -352,10 +354,13 @@ Requires: %{name}-base-libs = %{version}-%{release} %if 0%{?el7} Requires: llvm7.0-libs Requires: vespa-icu >= 65.1.0-1 -Requires: vespa-openssl >= 1.1.1k-1 Requires: vespa-protobuf = 3.17.3 %else Requires: libicu +%endif +%if 0%{?el7} || 0%{?el8} +Requires: vespa-openssl >= 1.1.1l-1 +%else Requires: openssl-libs %endif %if 0%{?el8} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 6cc5768e1bc..7e9c1be4047 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -103,14 +103,14 @@ public class Flags { public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2021-09-01", + List.of("vekterli"), "2020-12-02", "2021-11-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( "hide-shared-routing-endpoint", false, - List.of("tokle", "bjormel"), "2020-12-02", "2021-09-01", + List.of("tokle", "bjormel"), "2020-12-02", "2021-11-01", "Whether the controller should hide shared routing layer endpoint", "Takes effect immediately", APPLICATION_ID @@ -171,7 +171,7 @@ public class Flags { public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2021-09-01", + List.of("vekterli"), "2021-02-19", "2021-11-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -179,21 +179,21 @@ public class Flags { public static final UnboundIntFlag NUM_DISTRIBUTOR_STRIPES = defineIntFlag( "num-distributor-stripes", 0, - List.of("geirst", "vekterli"), "2021-04-20", "2021-09-01", + List.of("geirst", "vekterli"), "2021-04-20", "2021-11-01", "Specifies the number of stripes used by the distributor. When 0, legacy single stripe behavior is used.", "Takes effect after distributor restart", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( "max-concurrent-merges-per-node", 16, - List.of("balder", "vekterli"), "2021-06-06", "2021-09-01", + List.of("balder", "vekterli"), "2021-06-06", "2021-11-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( "max-merge-queue-size", 1024, - List.of("balder", "vekterli"), "2021-06-06", "2021-09-01", + List.of("balder", "vekterli"), "2021-06-06", "2021-11-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); @@ -227,7 +227,7 @@ public class Flags { public static final UnboundBooleanFlag REQUIRE_CONNECTIVITY_CHECK = defineFeatureFlag( "require-connectivity-check", true, - List.of("arnej"), "2021-06-03", "2021-09-01", + List.of("arnej"), "2021-06-03", "2021-12-01", "Require that config-sentinel connectivity check passes with good quality before starting services", "Takes effect on next restart", ZONE_ID, APPLICATION_ID); @@ -239,13 +239,6 @@ public class Flags { "Takes effect on next deployment through controller", APPLICATION_ID); - public static final UnboundBooleanFlag DRY_RUN_ONNX_ON_SETUP = defineFeatureFlag( - "dry-run-onnx-on-setup", true, - List.of("baldersheim"), "2021-06-23", "2021-09-01", - "Whether to dry run onnx models on setup for better error checking", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag USE_REAL_RESOURCES = defineFeatureFlag( "use-real-resources", false, List.of("freva"), "2021-09-08", "2021-10-01", |