diff options
Diffstat (limited to 'client/go/cmd/test.go')
-rw-r--r-- | client/go/cmd/test.go | 128 |
1 files changed, 72 insertions, 56 deletions
diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index dc75ca22729..b5399bd213a 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -53,10 +53,10 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, } exitFunc(3) } else if count == 0 { - fmt.Fprintf(stdout, "Failed to find any tests at '%v'\n", testPath) + fmt.Fprintf(stdout, "Failed to find any tests at %v\n", testPath) exitFunc(3) } else { - fmt.Fprintf(stdout, "%d tests completed successfully\n", count) + fmt.Fprintf(stdout, "\n%d tests completed successfully\n", count) } }, } @@ -71,12 +71,18 @@ func runTests(rootPath string, target vespa.Target) (int, []string) { if err != nil { fatalErr(err, "Failed reading specified test directory") } + previousFailed := false for _, test := range tests { if !test.IsDir() && filepath.Ext(test.Name()) == ".json" { testPath := path.Join(rootPath, test.Name()) + if previousFailed { + fmt.Fprintln(stdout, "") + previousFailed = false + } failure := runTest(testPath, target) if failure != "" { failed = append(failed, failure) + previousFailed = true } count++ } @@ -96,10 +102,10 @@ func runTest(testPath string, target vespa.Target) string { var test test testBytes, err := ioutil.ReadFile(testPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read test file at '%s'", testPath)) + fatalErr(err, fmt.Sprintf("Failed to read test file at %s", testPath)) } if err = json.Unmarshal(testBytes, &test); err != nil { - fatalErr(err, fmt.Sprintf("Failed to parse test file at '%s", testPath)) + fatalErr(err, fmt.Sprintf("Failed to parse test file at %s", testPath)) } testName := test.Name @@ -110,44 +116,44 @@ func runTest(testPath string, target vespa.Target) string { defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath)) if err != nil { - fatalErr(err, fmt.Sprintf("Invalid default parameters for '%s'", testName)) + fatalErr(err, fmt.Sprintf("Invalid default parameters for %s", testName)) } - if len(test.Assertions) == 0 { - fatalErr(fmt.Errorf("a test must have at least one assertion, but none were found in '%s'", testPath)) + if len(test.Steps) == 0 { + fatalErr(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath)) } - for i, assertion := range test.Assertions { - assertionName := assertion.Name - if assertionName == "" { - assertionName = fmt.Sprintf("assertion %d", i) + for i, step := range test.Steps { + stepName := step.Name + if stepName == "" { + stepName = fmt.Sprintf("step %d", i + 1) } - failure, err := verify(assertion, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target) + failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target) if err != nil { - fatalErr(err, fmt.Sprintf("\nError verifying %s", assertionName)) + fatalErr(err, fmt.Sprintf("Error in %s", stepName)) } if failure != "" { - fmt.Fprintf(stdout, "\nFailed verifying %s:\n%s\n", assertionName, failure) - return fmt.Sprintf("%v: %v", testName, assertionName) + fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure) + return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) } if i == 0 { fmt.Fprintf(stdout, " ") } fmt.Fprint(stdout, ".") } - fmt.Fprintln(stdout, " OK!") + fmt.Fprintln(stdout, " OK") return "" } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails -func verify(assertion assertion, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, error) { - requestBody, err := getBody(assertion.Request.BodyRaw, testsPath) +func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, string, error) { + requestBody, err := getBody(step.Request.BodyRaw, testsPath) if err != nil { - return "", err + return "", "", err } - parameters, err := getParameters(assertion.Request.ParametersRaw, testsPath) + parameters, err := getParameters(step.Request.ParametersRaw, testsPath) if err != nil { - return "", err + return "", "", err } for name, value := range defaultParameters { if _, present := parameters[name]; !present { @@ -155,28 +161,28 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul } } - cluster := assertion.Request.Cluster + cluster := step.Request.Cluster if cluster == "" { cluster = defaultCluster } service, err := target.Service("query", 0, 0, cluster) if err != nil { - return "", err + return "", "", err } - method := assertion.Request.Method + method := step.Request.Method if method == "" { method = "GET" } - pathAndQuery := assertion.Request.URI + pathAndQuery := step.Request.URI if pathAndQuery == "" { pathAndQuery = "/search/" } requestUrl, err := url.ParseRequestURI(service.BaseURL + pathAndQuery) if err != nil { - return "", err + return "", "", err } query := requestUrl.Query() for name, value := range parameters { @@ -197,50 +203,56 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul response, err := service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout if err != nil { - return "", err + return "", "", err } defer response.Body.Close() - statusCode := assertion.Response.Code + statusCode := step.Response.Code if statusCode == 0 { statusCode = 200 } if statusCode != response.StatusCode { - return fmt.Sprintf("Expected status code (%d) does not match actual (%d). Response body:\n%s", statusCode, response.StatusCode, util.ReaderToJSON(response.Body)), nil + failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode) + return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil } - responseBodySpecBytes, err := getBody(assertion.Response.BodyRaw, testsPath) + responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath) if err != nil { - return "", err + return "", "", err } if responseBodySpecBytes == nil { - return "", nil + return "", "", nil } var responseBodySpec interface{} err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) if err != nil { - return "", err + return "", "", err } responseBodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { - return "", err + return "", "", err } var responseBody interface{} err = json.Unmarshal(responseBodyBytes, &responseBody) if err != nil { - return "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) + return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) } - failure, err := compare(responseBodySpec, responseBody, "") + failure, expected, err := compare(responseBodySpec, responseBody, "") if failure != "" { responsePretty, _ := json.MarshalIndent(responseBody, "", " ") - failure = failure + " Response body:\n" + string(responsePretty) + longFailure := failure + if expected != "" { + longFailure += "\n" + expected + } + longFailure += "\nActual response:\n" + string(responsePretty) + return failure, longFailure, err } - return failure, err + return "", "", err } -func compare(expected interface{}, actual interface{}, path string) (string, error) { +func compare(expected interface{}, actual interface{}, path string) (string, string, error) { typeMatch := false valueMatch := false switch u := expected.(type) { @@ -265,14 +277,14 @@ func compare(expected interface{}, actual interface{}, path string) (string, err if ok { if len(u) == len(v) { for i, e := range u { - result, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)) - if result != "" || err != nil { - return result, err + failure, expected, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)) + if failure != "" || err != nil { + return failure, expected, err } } valueMatch = true } else { - return fmt.Sprintf("Expected number of elements at %s (%d) does not match actual (%d).", path, len(u), len(v)), nil + return fmt.Sprintf("Unexpected number of elements at %s: %d", path, len(v)), fmt.Sprintf("Expected: %d", len(u)), nil } } case map[string]interface{}: @@ -283,28 +295,32 @@ func compare(expected interface{}, actual interface{}, path string) (string, err childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1")) f, ok := v[n] if !ok { - return fmt.Sprintf("Expected field at %s not present in actual data.", childPath), nil + return fmt.Sprintf("Missing expected field at %s", childPath), "", nil } - result, err := compare(e, f, childPath) - if result != "" || err != nil { - return result, err + failure, expected, err := compare(e, f, childPath) + if failure != "" || err != nil { + return failure, expected, err } } valueMatch = true } default: - return "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected) + return "", "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected) } - if !(typeMatch && valueMatch) { + if !valueMatch { if path == "" { path = "root" } - expectedJson, _ := json.MarshalIndent(expected, "", " ") - actualJson, _ := json.MarshalIndent(actual, "", " ") - return fmt.Sprintf("Expected JSON at %s (%s) does not match actual (%s).", path, expectedJson, actualJson), nil + mismatched := "type" + if typeMatch { + mismatched = "value" + } + expectedJson, _ := json.Marshal(expected) + actualJson, _ := json.Marshal(actual) + return fmt.Sprintf("Unexpected %s at %s: %s", mismatched, path, actualJson), fmt.Sprintf("Expected: %s", expectedJson), nil } - return "", nil + return "", "", nil } func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) { @@ -340,8 +356,8 @@ func getBody(bodyRaw []byte, testsPath string) ([]byte, error) { type test struct { Name string `json:"name"` - Defaults defaults `json:"defaults"` - Assertions []assertion `json:"assertions"` + Defaults defaults `json:"defaults"` + Steps []step `json:"steps"` } type defaults struct { @@ -349,7 +365,7 @@ type defaults struct { ParametersRaw json.RawMessage `json:"parameters"` } -type assertion struct { +type step struct { Name string `json:"name"` Request request `json:"request"` Response response `json:"response"` |