diff options
22 files changed, 263 insertions, 126 deletions
diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index b465a3b8add..c94faa0d5e3 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -81,6 +81,6 @@ func printPublicKey(apiKeyFile, tenant string) { log.Printf("\nThis is your public key:\n%s", color.Green(pemPublicKey)) log.Printf("Its fingerprint is:\n%s\n", color.Cyan(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") - log.Printf(color.Cyan("https://console.vespa.oath.cloud/tenant/%s/keys").String(), tenant) + log.Printf(color.Cyan("%s/tenant/%s/keys").String(), defaultConsoleURL, tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index f44d3aeae42..19fa08ebaa4 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -61,13 +61,21 @@ If application directory is not specified, it defaults to working directory.`, opts.APIKey = readAPIKey(deployment.Application.Tenant) opts.Deployment = deployment } - if err := vespa.Deploy(opts); err == nil { - printSuccess("Deployed ", color.Cyan(pkg.Path)) + if sessionOrRunID, err := vespa.Deploy(opts); err == nil { if opts.IsCloud() { - log.Printf("\nUse %s for deployment status, or see", color.Cyan("vespa status")) - log.Print(color.Cyan(fmt.Sprintf("https://console.vespa.oath.cloud/tenant/%s/application/%s/dev/instance/%s", opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance))) + printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) + } else { + printSuccess("Deployed ", color.Cyan(pkg.Path)) } - waitForQueryService() + if opts.IsCloud() { + log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) + log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", + defaultConsoleURL, + opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance, + opts.Deployment.Zone.Environment, opts.Deployment.Zone.Region, + sessionOrRunID))) + } + waitForQueryService(sessionOrRunID) } else { fatalErr(nil, err.Error()) } @@ -123,17 +131,17 @@ var activateCmd = &cobra.Command{ }) if err == nil { printSuccess("Activated ", color.Cyan(pkg.Path), " with session ", sessionID) - waitForQueryService() + waitForQueryService(sessionID) } else { fatalErr(nil, err.Error()) } }, } -func waitForQueryService() { +func waitForQueryService(sessionOrRunID int64) { if waitSecsArg > 0 { log.Println() - waitForService("query") + waitForService("query", sessionOrRunID) } } diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index 8a438ad4125..450f061f140 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -107,7 +107,7 @@ var documentGetCmd = &cobra.Command{ }, } -func documentService() *vespa.Service { return getService("document") } +func documentService() *vespa.Service { return getService("document", 0) } func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) { if !result.Success { diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index 6e13c68f311..59af042e611 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -93,7 +93,7 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", executeCommand(t, client, arguments, []string{})) - target := getService("document").BaseURL + target := getService("document", 0).BaseURL expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) assert.Equal(t, target+"/document/v1/"+expectedPath, client.lastRequest.URL.String()) assert.Equal(t, "application/json", client.lastRequest.Header.Get("Content-Type")) @@ -115,7 +115,7 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { } `, executeCommand(t, client, arguments, []string{})) - target := getService("document").BaseURL + target := getService("document", 0).BaseURL expectedPath, _ := vespa.IdToURLPath(documentId) assert.Equal(t, target+"/document/v1/"+expectedPath, client.lastRequest.URL.String()) assert.Equal(t, "GET", client.lastRequest.Method) diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 67124ec2417..b672419cae6 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -17,6 +17,8 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) +const defaultConsoleURL = "https://console.vespa.oath.cloud" + var exitFunc = os.Exit // To allow overriding Exit in tests func fatalErrHint(err error, hints ...string) { @@ -94,13 +96,13 @@ func getTargetType() string { return target } -func getService(service string) *vespa.Service { +func getService(service string, sessionOrRunID int64) *vespa.Service { t := getTarget() timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { log.Printf("Waiting up to %d %s for services to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) } - if err := t.DiscoverServices(timeout); err != nil { + if err := t.DiscoverServices(timeout, sessionOrRunID); err != nil { fatalErr(err, "Services unavailable") } s, err := t.Service(service) @@ -134,8 +136,8 @@ func getTarget() vespa.Target { return nil } -func waitForService(service string) { - s := getService(service) +func waitForService(service string, sessionOrRunID int64) { + s := getService(service, sessionOrRunID) timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { log.Printf("Waiting up to %d %s for service to become ready ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index 062ea8d1bbc..ea80c037721 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -36,7 +36,7 @@ can be set by the syntax [parameter-name]=[value].`, } func query(arguments []string) { - service := getService("query") + service := getService("query", 0) url, _ := url.Parse(service.BaseURL + "/search/") urlQuery := url.Query() for i := 0; i < len(arguments); i++ { diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index 9bb5f07c3a1..af9f9c4cfd5 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -49,7 +49,7 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { "{\n \"query\": \"result\"\n}\n", executeCommand(t, client, []string{"query"}, query), "query output") - assert.Equal(t, getService("query").BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) + assert.Equal(t, getService("query", 0).BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } func assertQueryNonJsonResult(t *testing.T, expectedQuery string, query ...string) { @@ -58,7 +58,7 @@ func assertQueryNonJsonResult(t *testing.T, expectedQuery string, query ...strin "query result\n", executeCommand(t, client, []string{"query"}, query), "query output") - assert.Equal(t, getService("query").BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) + assert.Equal(t, getService("query", 0).BaseURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) } func assertQueryError(t *testing.T, status int, errorMessage string) { diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index 796ede9c86d..5fdcaa07d8a 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -22,7 +22,7 @@ var statusCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - waitForService("query") + waitForService("query", 0) }, } @@ -33,7 +33,7 @@ var statusQueryCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { - waitForService("query") + waitForService("query", 0) }, } @@ -44,7 +44,7 @@ var statusDocumentCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { - waitForService("document") + waitForService("document", 0) }, } @@ -55,6 +55,6 @@ var statusDeployCmd = &cobra.Command{ DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { - waitForService("deploy") + waitForService("deploy", 0) }, } diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index c2a27442a53..081e9fc17d2 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -198,17 +198,17 @@ func Activate(sessionID int64, deployment DeploymentOpts) error { return checkResponse(req, response, serviceDescription) } -func Deploy(opts DeploymentOpts) error { +func Deploy(opts DeploymentOpts) (int64, error) { path := "/application/v2/tenant/default/prepareandactivate" if opts.IsCloud() { if !opts.ApplicationPackage.HasCertificate() { - return fmt.Errorf("%s: missing certificate in package", opts) + return 0, fmt.Errorf("%s: missing certificate in package", opts) } if opts.APIKey == nil { - return fmt.Errorf("%s: missing api key", opts.String()) + return 0, fmt.Errorf("%s: missing api key", opts.String()) } if opts.Deployment.Zone.Environment == "" || opts.Deployment.Zone.Region == "" { - return fmt.Errorf("%s: missing zone", opts) + return 0, fmt.Errorf("%s: missing zone", opts) } path = fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/deploy/%s-%s", opts.Deployment.Application.Tenant, @@ -219,10 +219,9 @@ func Deploy(opts DeploymentOpts) error { } u, err := opts.url(path) if err != nil { - return err + return 0, err } - _, err = uploadApplicationPackage(u, opts) - return err + return uploadApplicationPackage(u, opts) } func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) { @@ -251,16 +250,20 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) } defer response.Body.Close() - var sessionResponse struct { - SessionID string `json:"session-id"` + var jsonResponse struct { + SessionID string `json:"session-id"` // Config server + RunID int64 `json:"run"` // Controller } - sessionResponse.SessionID = "0" // Set a default session ID for responses that don't contain int (e.g. cloud deployment) + jsonResponse.SessionID = "0" // Set a default session ID for responses that don't contain int (e.g. cloud deployment) if err := checkResponse(request, response, serviceDescription); err != nil { return 0, err } jsonDec := json.NewDecoder(response.Body) - jsonDec.Decode(&sessionResponse) // Ignore error in case this is a non-JSON response - return strconv.ParseInt(sessionResponse.SessionID, 10, 64) + jsonDec.Decode(&jsonResponse) // Ignore error in case this is a non-JSON response + if jsonResponse.RunID > 0 { + return jsonResponse.RunID, nil + } + return strconv.ParseInt(jsonResponse.SessionID, 10, 64) } func checkResponse(req *http.Request, response *http.Response, serviceDescription string) error { diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 493f3ec3485..faf80736293 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -41,8 +41,9 @@ type Target interface { // Service returns the service for given name. Service(name string) (*Service, error) - // DiscoverServices queries for services available on this target until one is found or timeout passes. - DiscoverServices(timeout time.Duration) error + // DiscoverServices queries for services available on this target after the given session or deployment run has + // completed. + DiscoverServices(timeout time.Duration, sessionOrRunID int64) error } type customTarget struct { @@ -75,9 +76,8 @@ func (s *Service) Wait(timeout time.Duration) (int, error) { if err != nil { return 0, err } - okFunc := func(status int, response []byte) (string, bool) { return "", status/100 == 2 } - status, _, err := wait(okFunc, req, s.certificate, timeout) - return status, err + okFunc := func(status int, response []byte) (bool, error) { return status/100 == 2, nil } + return wait(okFunc, req, s.certificate, timeout) } func (s *Service) Description() string { @@ -103,7 +103,7 @@ func (t *customTarget) Service(name string) (*Service, error) { return nil, fmt.Errorf("unknown service: %s", name) } -func (t *customTarget) DiscoverServices(timeout time.Duration) error { return nil } +func (t *customTarget) DiscoverServices(timeout time.Duration, sessionID int64) error { return nil } func (t *localTarget) Type() string { return t.targetType } @@ -117,7 +117,7 @@ func (t *localTarget) Service(name string) (*Service, error) { return nil, fmt.Errorf("unknown service: %s", name) } -func (t *localTarget) DiscoverServices(timeout time.Duration) error { return nil } +func (t *localTarget) DiscoverServices(timeout time.Duration, sessionID int64) error { return nil } type cloudTarget struct { cloudAPI string @@ -150,39 +150,85 @@ func (t *cloudTarget) Service(name string) (*Service, error) { return nil, fmt.Errorf("unknown service: %s", name) } -// DiscoverServices queries Vespa Cloud for endpoints until at least one endpoint is returned, or timeout passes. -func (t *cloudTarget) DiscoverServices(timeout time.Duration) error { +// DiscoverServices waits for run identified by runID to complete and at least one endpoint is available, or timeout +// passes. +func (t *cloudTarget) DiscoverServices(timeout time.Duration, runID int64) error { + signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) + if runID > 0 { + if err := t.waitForRun(signer, runID, timeout); err != nil { + return err + } + } + return t.discoverEndpoints(signer, timeout) +} + +func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout time.Duration) error { + runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d", + t.cloudAPI, + t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, + t.deployment.Zone.Environment, t.deployment.Zone.Region, runID) + req, err := http.NewRequest("GET", runURL, nil) + if err != nil { + return err + } + if err := signer.SignRequest(req); err != nil { + return err + } + jobSuccessFunc := func(status int, response []byte) (bool, error) { + if status/100 != 2 { + return false, nil + } + var resp jobResponse + if err := json.Unmarshal(response, &resp); err != nil { + return false, nil + } + if resp.Active { + return false, nil + } + if resp.Status != "success" { + return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status) + } + return true, nil + } + _, err = wait(jobSuccessFunc, req, t.keyPair, timeout) + return err +} + +func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Duration) error { deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s", t.cloudAPI, t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, t.deployment.Zone.Environment, t.deployment.Zone.Region) req, err := http.NewRequest("GET", deploymentURL, nil) - signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) + if err != nil { + return err + } if err := signer.SignRequest(req); err != nil { return err } - endpointFunc := func(status int, response []byte) (string, bool) { + var endpointURL string + endpointFunc := func(status int, response []byte) (bool, error) { if status/100 != 2 { - return "", false + return false, nil } var resp deploymentResponse if err := json.Unmarshal(response, &resp); err != nil { - return "", false + return false, nil } if len(resp.Endpoints) == 0 { - return "", false + return false, nil } - return resp.Endpoints[0].URL, true + endpointURL = resp.Endpoints[0].URL + return true, nil } - _, endpoint, err := wait(endpointFunc, req, t.keyPair, timeout) - if err != nil { + if _, err = wait(endpointFunc, req, t.keyPair, timeout); err != nil { return err } - if endpoint == "" { + if endpointURL == "" { return fmt.Errorf("no endpoint discovered") } - t.queryURL = endpoint - t.documentURL = endpoint + t.queryURL = endpointURL + t.documentURL = endpointURL return nil } @@ -213,9 +259,14 @@ type deploymentResponse struct { Endpoints []deploymentEndpoint `json:"endpoints"` } -type responseFunc func(status int, response []byte) (string, bool) +type jobResponse struct { + Active bool `json:"active"` + Status string `json:"status"` +} -func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeout time.Duration) (int, string, error) { +type responseFunc func(status int, response []byte) (bool, error) + +func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeout time.Duration) (int, error) { if certificate.Certificate != nil { util.ActiveHttpClient.UseCertificate(certificate) } @@ -232,12 +283,15 @@ func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeo statusCode = response.StatusCode body, err := ioutil.ReadAll(response.Body) if err != nil { - return 0, "", err + return 0, err } response.Body.Close() - result, ok := fn(statusCode, body) + ok, err := fn(statusCode, body) + if err != nil { + return statusCode, err + } if ok { - return statusCode, result, nil + return statusCode, nil } } if loopOnce { @@ -245,5 +299,5 @@ func wait(fn responseFunc, req *http.Request, certificate tls.Certificate, timeo } time.Sleep(waitRetryInterval) } - return statusCode, "", httpErr + return statusCode, httpErr } diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index bd9a93d9d24..1f46cc83178 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -23,6 +23,14 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request response = fmt.Sprintf(`{"endpoints": [{"url": "%s"}]}`, v.serverURL) } w.Write([]byte(response)) + case "/application/v4/tenant/t1/application/a1/instance/i1/job/dev-us-north-1/run/42": + response := "{}" + if v.endpointsReady { + response = `{"active": false, "status": "success"}` + } else { + response = `{"active": true, "status": "running"}` + } + w.Write([]byte(response)) case "/status.html": w.Write([]byte("OK")) case "/ApplicationStatus": @@ -76,11 +84,11 @@ func TestCloudTargetWait(t *testing.T) { _, err = target.Service("query") assert.NotNil(t, err) - err = target.DiscoverServices(0) + err = target.DiscoverServices(0, 42) assert.NotNil(t, err) vc.endpointsReady = true - err = target.DiscoverServices(0) + err = target.DiscoverServices(0, 42) assert.Nil(t, err) assertServiceWait(t, 500, target, "query") diff --git a/container-core/src/main/sh/vespa-jvm-dumper b/container-core/src/main/sh/vespa-jvm-dumper index 22d5fa0f98a..50f4a1b2d63 100755 --- a/container-core/src/main/sh/vespa-jvm-dumper +++ b/container-core/src/main/sh/vespa-jvm-dumper @@ -83,51 +83,64 @@ if [ $# -ne 2 ]; then exit 1 fi -readonly SERVICE=$1 -readonly OUTPUT_DIRECTORY=$2 -if ! [ -d "${OUTPUT_DIRECTORY}" -a -w "${OUTPUT_DIRECTORY}" ]; then - echo "Directory '${OUTPUT_DIRECTORY}' is not writable" +readonly service=$1 +readonly output_directory=$2 +if ! [ -d "${output_directory}" -a -w "${output_directory}" ]; then + echo "Directory '${output_directory}' is not writable" exit 1 fi -readonly STATUS=$(vespa-sentinel-cmd list | grep "id=\"${SERVICE}\"") -if [ -z "${STATUS}" ]; then - echo "No service named '${SERVICE}'" +readonly status=$(vespa-sentinel-cmd list | grep "id=\"${service}\"") +if [ -z "${status}" ]; then + echo "No service named '${service}'" exit 1 else - echo "Found service: ${STATUS}" + echo "Found service: ${status}" fi -readonly JVM_PID=$(echo ${STATUS} | cut -d " " -f 4 | cut -d "=" -f 2) -if ! [[ "${JVM_PID}" =~ ^[0-9]+$ ]]; then - echo "Could not find valid pid for '${SERVICE}' (pid='${JVM_PID}')" +readonly jvm_pid=$(echo ${status} | cut -d " " -f 4 | cut -d "=" -f 2) +if ! [[ "${jvm_pid}" =~ ^[0-9]+$ ]]; then + echo "Could not find valid pid for '${service}' (pid='${jvm_pid}')" exit 1 else - echo "Pid for '${SERVICE}' is '${JVM_PID}'" + echo "Pid for '${service}' is '${jvm_pid}'" fi -if ! [ -n $(ps -p ${JVM_PID} -o pid=) ]; then - echo "Could not find process for '${JVM_PID}'" +if ! [ -n $(ps -p ${jvm_pid} -o pid=) ]; then + echo "Could not find process for '${jvm_pid}'" exit 1 fi +echo "Starting Java Flight Recorder recording" +jcmd ${jvm_pid} JFR.start name=vespa-jvm-dumper path-to-gc-roots=true settings=profile filename=${output_directory}/jvm-jfr-dump.jfr + +readonly sleep_seconds=15 +echo "Waiting ${sleep_seconds} before stopping Java Flight Recorder" +sleep ${sleep_seconds}s + +echo "Dumping Java Flight Recorder recording to file" +jcmd ${jvm_pid} JFR.dump name=vespa-jvm-dumper + echo "Creating heap dump" -readonly HEAP_DUMP_FILE=${OUTPUT_DIRECTORY}/jvm-heap-dump.hprof -if test -f "${HEAP_DUMP_FILE}"; then - rm "${HEAP_DUMP_FILE}" +readonly heap_dump_file=${output_directory}/jvm-heap-dump.hprof +if test -f "${heap_dump_file}"; then + rm "${heap_dump_file}" fi -jmap -dump:live,format=b,file=${HEAP_DUMP_FILE} ${JVM_PID} +jmap -dump:live,format=b,file=${heap_dump_file} ${jvm_pid} + +echo "Trigger vespa-malloc to dump information to Vespa log" +kill -SIGPROF ${jvm_pid} echo "Getting jmap information" -jhsdb jmap --heap --pid ${JVM_PID} > ${OUTPUT_DIRECTORY}/jmap-output.txt +jhsdb jmap --heap --pid ${jvm_pid} > ${output_directory}/jmap-output.txt echo "Getting jstat information" -jstat -gcutil ${JVM_PID} > ${OUTPUT_DIRECTORY}/jstat-output.txt +jstat -gcutil ${jvm_pid} > ${output_directory}/jstat-output.txt echo "Getting jstack information" -jstack ${JVM_PID} > ${OUTPUT_DIRECTORY}/jstack-output.txt +jstack ${jvm_pid} > ${output_directory}/jstack-output.txt echo "Getting pmap information" -pmap -x ${JVM_PID} | sort -nk3 | tail -10 > ${OUTPUT_DIRECTORY}/pmap-output.txt +pmap -x ${jvm_pid} | sort -nk3 | tail -10 > ${output_directory}/pmap-output.txt echo "Copying vespa logs" -cp ${VESPA_HOME}/logs/vespa/vespa.log ${OUTPUT_DIRECTORY}/vespa.log +cp ${VESPA_HOME}/logs/vespa/vespa.log ${output_directory}/vespa.log echo "Done!" diff --git a/dist/vespa.spec b/dist/vespa.spec index a66fc08038e..ef20e7eb8eb 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -285,7 +285,7 @@ Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private # _dl_sym function. # Exclude automated requires for libraries in /opt/vespa-deps/lib64. -%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ +%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ %description diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 233910cc3d1..e0c75b99d83 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,7 +13,9 @@ import java.util.Optional; import java.util.TreeMap; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; +import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; @@ -108,7 +110,7 @@ public class Flags { public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( "hide-shared-routing-endpoint", false, - List.of("tokle", "bjormel"), "2020-12-02", "2021-09-01", + List.of("tokle", "bjormel"), "2020-12-02", "2021-11-01", "Whether the controller should hide shared routing layer endpoint", "Takes effect immediately", APPLICATION_ID @@ -183,7 +185,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( - "max-concurrent-merges-per-node", 16, + "max-concurrent-merges-per-node", 128, List.of("balder", "vekterli"), "2021-06-06", "2021-11-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", @@ -237,6 +239,13 @@ public class Flags { "Takes effect on next deployment through controller", APPLICATION_ID); + public static final UnboundBooleanFlag USE_REAL_RESOURCES = defineFeatureFlag( + "use-real-resources", false, + List.of("freva"), "2021-09-08", "2021-10-01", + "Whether host-admin should use real resources (rather than advertised resources) when creating linux container and reporting metrics", + "Takes effect on next host-admin tick", + CLUSTER_TYPE, NODE_TYPE); + public static final UnboundListFlag<String> DEFER_APPLICATION_ENCRYPTION = defineListFlag( "defer-application-encryption", List.of(), String.class, List.of("mpolden", "hakonhall"), "2021-06-23", "2021-10-01", diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java index 7d52b9d72b0..061a06f4687 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver; import com.yahoo.vespa.flags.FlagRepository; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.node.admin.configserver.flags.RealFlagRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; @@ -26,9 +27,9 @@ public class RealConfigServerClients implements ConfigServerClients { /** * @param configServerApi the backend API to use - will be closed at {@link #stop()}. */ - public RealConfigServerClients(ConfigServerApi configServerApi) { + public RealConfigServerClients(ConfigServerApi configServerApi, FlagSource flagSource) { this.configServerApi = configServerApi; - nodeRepository = new RealNodeRepository(configServerApi); + nodeRepository = new RealNodeRepository(configServerApi, flagSource); orchestrator = new OrchestratorImpl(configServerApi); state = new StateImpl(configServerApi); flagRepository = new RealFlagRepository(configServerApi); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index fa1f8528b31..30bc1ef5ea3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -55,6 +55,7 @@ public class NodeSpec { private final Optional<NodeMembership> membership; private final NodeResources resources; + private final NodeResources realResources; private final Set<String> ipAddresses; private final Set<String> additionalIpAddresses; @@ -88,6 +89,7 @@ public class NodeSpec { Optional<Instant> currentFirmwareCheck, Optional<String> modelName, NodeResources resources, + NodeResources realResources, Set<String> ipAddresses, Set<String> additionalIpAddresses, NodeReports reports, @@ -125,6 +127,7 @@ public class NodeSpec { this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck); this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck); this.resources = Objects.requireNonNull(resources); + this.realResources = Objects.requireNonNull(realResources); this.ipAddresses = Objects.requireNonNull(ipAddresses); this.additionalIpAddresses = Objects.requireNonNull(additionalIpAddresses); this.reports = Objects.requireNonNull(reports); @@ -222,28 +225,32 @@ public class NodeSpec { return resources; } + public NodeResources realResources() { + return realResources; + } + public double vcpu() { - return resources.vcpu(); + return realResources.vcpu(); } public double memoryGb() { - return resources.memoryGb(); + return realResources.memoryGb(); } public DiskSize diskSize() { - return DiskSize.of(resources.diskGb(), DiskSize.Unit.GB); + return DiskSize.of(realResources.diskGb(), DiskSize.Unit.GB); } public double diskGb() { - return resources.diskGb(); + return realResources.diskGb(); } public boolean isFastDisk() { - return resources.diskSpeed() == fast; + return realResources.diskSpeed() == fast; } public double bandwidthGbps() { - return resources.bandwidthGbps(); + return realResources.bandwidthGbps(); } public Set<String> ipAddresses() { @@ -297,6 +304,7 @@ public class NodeSpec { Objects.equals(wantedFirmwareCheck, that.wantedFirmwareCheck) && Objects.equals(currentFirmwareCheck, that.currentFirmwareCheck) && Objects.equals(resources, that.resources) && + Objects.equals(realResources, that.realResources) && Objects.equals(ipAddresses, that.ipAddresses) && Objects.equals(additionalIpAddresses, that.additionalIpAddresses) && Objects.equals(reports, that.reports) && @@ -330,6 +338,7 @@ public class NodeSpec { wantedFirmwareCheck, currentFirmwareCheck, resources, + realResources, ipAddresses, additionalIpAddresses, reports, @@ -363,6 +372,7 @@ public class NodeSpec { + " wantedFirmwareCheck=" + wantedFirmwareCheck + " currentFirmwareCheck=" + currentFirmwareCheck + " resources=" + resources + + " realResources=" + realResources + " ipAddresses=" + ipAddresses + " additionalIpAddresses=" + additionalIpAddresses + " reports=" + reports @@ -394,7 +404,8 @@ public class NodeSpec { private Optional<Instant> wantedFirmwareCheck = Optional.empty(); private Optional<Instant> currentFirmwareCheck = Optional.empty(); private Optional<String> modelName = Optional.empty(); - private NodeResources resources = new NodeResources(0, 0, 0, 0, slow); + private NodeResources resources; + private NodeResources realResources; private Set<String> ipAddresses = Set.of(); private Set<String> additionalIpAddresses = Set.of(); private NodeReports reports = new NodeReports(); @@ -410,6 +421,7 @@ public class NodeSpec { type(node.type); flavor(node.flavor); resources(node.resources); + realResources(node.realResources); ipAddresses(node.ipAddresses); additionalIpAddresses(node.additionalIpAddresses); wantedRebootGeneration(node.wantedRebootGeneration); @@ -538,24 +550,29 @@ public class NodeSpec { return this; } + public Builder realResources(NodeResources realResources) { + this.realResources = realResources; + return this; + } + public Builder vcpu(double vcpu) { - return resources(resources.withVcpu(vcpu)); + return realResources(realResources.withVcpu(vcpu)); } public Builder memoryGb(double memoryGb) { - return resources(resources.withMemoryGb(memoryGb)); + return realResources(realResources.withMemoryGb(memoryGb)); } public Builder diskGb(double diskGb) { - return resources(resources.withDiskGb(diskGb)); + return realResources(realResources.withDiskGb(diskGb)); } public Builder fastDisk(boolean fastDisk) { - return resources(resources.with(fastDisk ? fast : slow)); + return realResources(realResources.with(fastDisk ? fast : slow)); } public Builder bandwidthGbps(double bandwidthGbps) { - return resources(resources.withBandwidthGbps(bandwidthGbps)); + return realResources(realResources.withBandwidthGbps(bandwidthGbps)); } public Builder ipAddresses(Set<String> ipAddresses) { @@ -681,6 +698,10 @@ public class NodeSpec { return resources; } + public NodeResources realResources() { + return realResources; + } + public Set<String> ipAddresses() { return ipAddresses; } @@ -708,7 +729,7 @@ public class NodeSpec { wantedRestartGeneration, currentRestartGeneration, wantedRebootGeneration, currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, modelName, - resources, ipAddresses, additionalIpAddresses, + resources, realResources, ipAddresses, additionalIpAddresses, reports, parentHostname, archiveUri, exclusiveTo); } @@ -727,7 +748,8 @@ public class NodeSpec { .state(state) .type(NodeType.tenant) .flavor("d-2-8-50") - .resources(new NodeResources(2, 8, 50, 10)); + .resources(new NodeResources(2, 8, 50, 10)) + .realResources(new NodeResources(2, 8, 50, 10)); // Set the required allocated fields if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 8e069ab923b..8934100a463 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -9,6 +9,10 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.host.FlavorOverrides; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetAclResponse; @@ -37,9 +41,11 @@ public class RealNodeRepository implements NodeRepository { private static final Logger logger = Logger.getLogger(RealNodeRepository.class.getName()); private final ConfigServerApi configServerApi; + private final BooleanFlag useRealResourcesFlag; - public RealNodeRepository(ConfigServerApi configServerApi) { + public RealNodeRepository(ConfigServerApi configServerApi, FlagSource flagSource) { this.configServerApi = configServerApi; + this.useRealResourcesFlag = Flags.USE_REAL_RESOURCES.bindTo(flagSource); } @Override @@ -59,7 +65,7 @@ public class RealNodeRepository implements NodeRepository { final GetNodesResponse nodesForHost = configServerApi.get(path, GetNodesResponse.class); return nodesForHost.nodes.stream() - .map(RealNodeRepository::createNodeSpec) + .map(this::createNodeSpec) .collect(Collectors.toList()); } @@ -69,7 +75,7 @@ public class RealNodeRepository implements NodeRepository { NodeRepositoryNode nodeResponse = configServerApi.get("/nodes/v2/node/" + hostName, NodeRepositoryNode.class); - return Optional.ofNullable(nodeResponse).map(RealNodeRepository::createNodeSpec); + return Optional.ofNullable(nodeResponse).map(this::createNodeSpec); } catch (HttpException.NotFoundException | HttpException.ForbiddenException e) { // Return empty on 403 in addition to 404 as it likely means we're trying to access a node that // has been deleted. When a node is deleted, the parent-child relationship no longer exists and @@ -141,7 +147,7 @@ public class RealNodeRepository implements NodeRepository { throw new NodeRepositoryException("Failed to set node state: " + response.message + " " + response.errorCode); } - private static NodeSpec createNodeSpec(NodeRepositoryNode node) { + private NodeSpec createNodeSpec(NodeRepositoryNode node) { Objects.requireNonNull(node.type, "Unknown node type"); NodeType nodeType = NodeType.valueOf(node.type); @@ -151,6 +157,9 @@ public class RealNodeRepository implements NodeRepository { Optional<NodeMembership> membership = Optional.ofNullable(node.membership) .map(m -> new NodeMembership(m.clusterType, m.clusterId, m.group, m.index, m.retired)); NodeReports reports = NodeReports.fromMap(Optional.ofNullable(node.reports).orElseGet(Map::of)); + boolean useRealResources = useRealResourcesFlag.with(FetchVector.Dimension.CLUSTER_TYPE, membership.map(m -> m.type().value())) + .with(FetchVector.Dimension.NODE_TYPE, nodeType.name()) + .value(); return new NodeSpec( node.hostname, Optional.ofNullable(node.openStackId), @@ -173,13 +182,8 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.wantedFirmwareCheck).map(Instant::ofEpochMilli), Optional.ofNullable(node.currentFirmwareCheck).map(Instant::ofEpochMilli), Optional.ofNullable(node.modelName), - new NodeResources( - node.resources.vcpu, - node.resources.memoryGb, - node.resources.diskGb, - node.resources.bandwidthGbps, - diskSpeedFromString(node.resources.diskSpeed), - storageTypeFromString(node.resources.storageType)), + nodeResources(node.resources), + nodeResources(useRealResources ? node.realResources : node.resources), node.ipAddresses, node.additionalIpAddresses, reports, @@ -188,6 +192,16 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm)); } + private static NodeResources nodeResources(NodeRepositoryNode.NodeResources nodeResources) { + return new NodeResources( + nodeResources.vcpu, + nodeResources.memoryGb, + nodeResources.diskGb, + nodeResources.bandwidthGbps, + diskSpeedFromString(nodeResources.diskSpeed), + storageTypeFromString(nodeResources.storageType)); + } + private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) { if (diskSpeed == null) return NodeResources.DiskSpeed.getDefault(); switch (diskSpeed) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 988bd2a4bf3..86caab9bf51 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -30,6 +30,8 @@ public class NodeRepositoryNode { public String flavor; @JsonProperty("resources") public NodeResources resources; + @JsonProperty("realResources") + public NodeResources realResources; @JsonProperty("membership") public Membership membership; @JsonProperty("owner") @@ -95,6 +97,7 @@ public class NodeRepositoryNode { ", modelName='" + modelName + '\'' + ", flavor='" + flavor + '\'' + ", resources=" + resources + + ", realResources=" + realResources + ", membership=" + membership + ", owner=" + owner + ", restartGeneration=" + restartGeneration + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index 06cc3f2c9f0..d8be6d1de7b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import java.net.URI; import java.nio.file.Path; import java.time.Instant; +import java.util.List; import java.util.Optional; /** @@ -66,7 +67,8 @@ public class SyncFileInfo { public static Optional<SyncFileInfo> forServiceDump(URI directory, Path file, Instant expiry) { String filename = file.getFileName().toString(); - Compression compression = filename.endsWith(".bin") || filename.endsWith(".hprof") ? Compression.ZSTD : Compression.NONE; + List<String> filesToCompress = List.of(".bin", ".hprof", ".jfr", ".log"); + Compression compression = filesToCompress.stream().anyMatch(filename::endsWith) ? Compression.ZSTD : Compression.NONE; if (filename.startsWith(".")) return Optional.empty(); URI location = directory.resolve(filename + compression.extension); return Optional.of(new SyncFileInfo(file, location, compression, expiry)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index fe06812c608..af88890f4a2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.host.FlavorOverrides; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; @@ -79,7 +80,7 @@ public class RealNodeRepositoryTest { private void waitForJdiscContainerToServe(ConfigServerApi configServerApi) throws InterruptedException { Instant start = Instant.now(); - nodeRepositoryApi = new RealNodeRepository(configServerApi); + nodeRepositoryApi = new RealNodeRepository(configServerApi, new InMemoryFlagSource()); while (Instant.now().minusSeconds(120).isBefore(start)) { try { nodeRepositoryApi.getNodes("foobar"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index beca554fb2d..2a217ff1c84 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -143,7 +143,7 @@ public class StorageMaintainerTest { @Test public void not_run_if_not_enough_used() throws IOException { NodeAgentContext context = new NodeAgentContextImpl.Builder( - NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) + NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); Files.createDirectories(context.pathOnHostFromPathInNode("/")); mockDiskUsage(500L); @@ -155,7 +155,7 @@ public class StorageMaintainerTest { @Test public void deletes_correct_amount() throws IOException { NodeAgentContext context = new NodeAgentContextImpl.Builder( - NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) + NodeSpec.Builder.testSpec("h123a.domain.tld").realResources(new NodeResources(1, 1, 1, 1)).build()) .fileSystem(fileSystem).build(); Files.createDirectories(context.pathOnHostFromPathInNode("/")); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 102dd5c9b1c..e7407c1bcc0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -697,13 +697,10 @@ public class NodeAgentImplTest { } private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { - NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() - .resources(resources) - .hostname(hostName) + NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) .type(NodeType.tenant) .flavor("docker") - .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) - .state(nodeState); + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage); owner.ifPresent(nodeBuilder::owner); NodeSpec node = nodeBuilder.build(); @@ -784,6 +781,6 @@ public class NodeAgentImplTest { } private NodeSpec.Builder nodeBuilder(NodeState state) { - return NodeSpec.Builder.testSpec(hostName, state).resources(resources); + return NodeSpec.Builder.testSpec(hostName, state).realResources(resources); } } |