From a151693a4a6b847744b7da55edde62c3413e6c75 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 24 Nov 2021 15:05:36 +0100 Subject: Format changes, and rename to steps array --- client/go/cmd/test.go | 120 +++++++++++---------- client/go/cmd/test_test.go | 8 +- client/go/cmd/testdata/tests/expected-suite.out | 113 +++++++++++++------ client/go/cmd/testdata/tests/expected.out | 3 +- client/go/cmd/testdata/tests/system-test/test.json | 3 +- .../tests/system-test/wrong-bool-value.json | 2 +- .../tests/system-test/wrong-element-count.json | 2 +- .../tests/system-test/wrong-field-name.json | 2 +- .../tests/system-test/wrong-float-value.json | 2 +- .../tests/system-test/wrong-int-value.json | 2 +- .../tests/system-test/wrong-null-value.json | 2 +- .../tests/system-test/wrong-string-value.json | 2 +- .../cmd/testdata/tests/system-test/wrong-type.json | 15 +++ 13 files changed, 178 insertions(+), 98 deletions(-) create mode 100644 client/go/cmd/testdata/tests/system-test/wrong-type.json (limited to 'client') diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index dc75ca22729..fa2e47ce5d3 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -53,7 +53,7 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, } exitFunc(3) } else if count == 0 { - fmt.Fprintf(stdout, "Failed to find any tests at '%v'\n", testPath) + fmt.Fprintf(stdout, "Failed to find any tests at %v\n", testPath) exitFunc(3) } else { fmt.Fprintf(stdout, "%d tests completed successfully\n", count) @@ -96,10 +96,10 @@ func runTest(testPath string, target vespa.Target) string { var test test testBytes, err := ioutil.ReadFile(testPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read test file at '%s'", testPath)) + fatalErr(err, fmt.Sprintf("Failed to read test file at %s", testPath)) } if err = json.Unmarshal(testBytes, &test); err != nil { - fatalErr(err, fmt.Sprintf("Failed to parse test file at '%s", testPath)) + fatalErr(err, fmt.Sprintf("Failed to parse test file at %s", testPath)) } testName := test.Name @@ -110,44 +110,44 @@ func runTest(testPath string, target vespa.Target) string { defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath)) if err != nil { - fatalErr(err, fmt.Sprintf("Invalid default parameters for '%s'", testName)) + fatalErr(err, fmt.Sprintf("Invalid default parameters for %s", testName)) } - if len(test.Assertions) == 0 { - fatalErr(fmt.Errorf("a test must have at least one assertion, but none were found in '%s'", testPath)) + if len(test.Steps) == 0 { + fatalErr(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath)) } - for i, assertion := range test.Assertions { - assertionName := assertion.Name - if assertionName == "" { - assertionName = fmt.Sprintf("assertion %d", i) + for i, step := range test.Steps { + stepName := step.Name + if stepName == "" { + stepName = fmt.Sprintf("step %d", i + 1) } - failure, err := verify(assertion, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target) + failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target) if err != nil { - fatalErr(err, fmt.Sprintf("\nError verifying %s", assertionName)) + fatalErr(err, fmt.Sprintf("Error in %s", stepName)) } if failure != "" { - fmt.Fprintf(stdout, "\nFailed verifying %s:\n%s\n", assertionName, failure) - return fmt.Sprintf("%v: %v", testName, assertionName) + fmt.Fprintf(stdout, " Failed %s:\n%s\n\n", stepName, longFailure) + return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) } if i == 0 { fmt.Fprintf(stdout, " ") } fmt.Fprint(stdout, ".") } - fmt.Fprintln(stdout, " OK!") + fmt.Fprintf(stdout, " OK\n\n") return "" } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails -func verify(assertion assertion, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, error) { - requestBody, err := getBody(assertion.Request.BodyRaw, testsPath) +func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, string, error) { + requestBody, err := getBody(step.Request.BodyRaw, testsPath) if err != nil { - return "", err + return "", "", err } - parameters, err := getParameters(assertion.Request.ParametersRaw, testsPath) + parameters, err := getParameters(step.Request.ParametersRaw, testsPath) if err != nil { - return "", err + return "", "", err } for name, value := range defaultParameters { if _, present := parameters[name]; !present { @@ -155,28 +155,28 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul } } - cluster := assertion.Request.Cluster + cluster := step.Request.Cluster if cluster == "" { cluster = defaultCluster } service, err := target.Service("query", 0, 0, cluster) if err != nil { - return "", err + return "", "", err } - method := assertion.Request.Method + method := step.Request.Method if method == "" { method = "GET" } - pathAndQuery := assertion.Request.URI + pathAndQuery := step.Request.URI if pathAndQuery == "" { pathAndQuery = "/search/" } requestUrl, err := url.ParseRequestURI(service.BaseURL + pathAndQuery) if err != nil { - return "", err + return "", "", err } query := requestUrl.Query() for name, value := range parameters { @@ -197,50 +197,56 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul response, err := service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout if err != nil { - return "", err + return "", "", err } defer response.Body.Close() - statusCode := assertion.Response.Code + statusCode := step.Response.Code if statusCode == 0 { statusCode = 200 } if statusCode != response.StatusCode { - return fmt.Sprintf("Expected status code (%d) does not match actual (%d). Response body:\n%s", statusCode, response.StatusCode, util.ReaderToJSON(response.Body)), nil + failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode) + return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil } - responseBodySpecBytes, err := getBody(assertion.Response.BodyRaw, testsPath) + responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath) if err != nil { - return "", err + return "", "", err } if responseBodySpecBytes == nil { - return "", nil + return "", "", nil } var responseBodySpec interface{} err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) if err != nil { - return "", err + return "", "", err } responseBodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { - return "", err + return "", "", err } var responseBody interface{} err = json.Unmarshal(responseBodyBytes, &responseBody) if err != nil { - return "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) + return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) } - failure, err := compare(responseBodySpec, responseBody, "") + failure, expected, err := compare(responseBodySpec, responseBody, "") if failure != "" { responsePretty, _ := json.MarshalIndent(responseBody, "", " ") - failure = failure + " Response body:\n" + string(responsePretty) + longFailure := failure + if expected != "" { + longFailure += "\n" + expected + } + longFailure += "\nActual response:\n" + string(responsePretty) + return failure, longFailure, err } - return failure, err + return "", "", err } -func compare(expected interface{}, actual interface{}, path string) (string, error) { +func compare(expected interface{}, actual interface{}, path string) (string, string, error) { typeMatch := false valueMatch := false switch u := expected.(type) { @@ -265,14 +271,14 @@ func compare(expected interface{}, actual interface{}, path string) (string, err if ok { if len(u) == len(v) { for i, e := range u { - result, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)) - if result != "" || err != nil { - return result, err + failure, expected, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)) + if failure != "" || err != nil { + return failure, expected, err } } valueMatch = true } else { - return fmt.Sprintf("Expected number of elements at %s (%d) does not match actual (%d).", path, len(u), len(v)), nil + return fmt.Sprintf("Unexpected number of elements at %s: %d", path, len(v)), fmt.Sprintf("Expected: %d", len(u)), nil } } case map[string]interface{}: @@ -283,28 +289,32 @@ func compare(expected interface{}, actual interface{}, path string) (string, err childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1")) f, ok := v[n] if !ok { - return fmt.Sprintf("Expected field at %s not present in actual data.", childPath), nil + return fmt.Sprintf("Missing expected field at %s", childPath), "", nil } - result, err := compare(e, f, childPath) - if result != "" || err != nil { - return result, err + failure, expected, err := compare(e, f, childPath) + if failure != "" || err != nil { + return failure, expected, err } } valueMatch = true } default: - return "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected) + return "", "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected) } - if !(typeMatch && valueMatch) { + if !valueMatch { if path == "" { path = "root" } - expectedJson, _ := json.MarshalIndent(expected, "", " ") - actualJson, _ := json.MarshalIndent(actual, "", " ") - return fmt.Sprintf("Expected JSON at %s (%s) does not match actual (%s).", path, expectedJson, actualJson), nil + mismatched := "type" + if typeMatch { + mismatched = "value" + } + expectedJson, _ := json.Marshal(expected) + actualJson, _ := json.Marshal(actual) + return fmt.Sprintf("Unexpected %s at %s: %s", mismatched, path, actualJson), fmt.Sprintf("Expected: %s", expectedJson), nil } - return "", nil + return "", "", nil } func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) { @@ -340,8 +350,8 @@ func getBody(bodyRaw []byte, testsPath string) ([]byte, error) { type test struct { Name string `json:"name"` - Defaults defaults `json:"defaults"` - Assertions []assertion `json:"assertions"` + Defaults defaults `json:"defaults"` + Steps []step `json:"steps"` } type defaults struct { @@ -349,7 +359,7 @@ type defaults struct { ParametersRaw json.RawMessage `json:"parameters"` } -type assertion struct { +type step struct { Name string `json:"name"` Request request `json:"request"` Response response `json:"response"` diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index 9d92e285750..8db10282d51 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -23,7 +23,7 @@ func TestSuite(t *testing.T) { searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json") client.NextStatus(200) client.NextStatus(200) - for i := 0; i < 9; i++ { + for i := 0; i < 10; i++ { client.NextResponse(200, string(searchResponse)) } @@ -35,7 +35,7 @@ func TestSuite(t *testing.T) { baseUrl := "http://127.0.0.1:8080" urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)} - for i := 0; i < 7; i++ { + for i := 0; i < 8; i++ { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) @@ -44,13 +44,13 @@ func TestSuite(t *testing.T) { func TestTestWithoutAssertions(t *testing.T) { client := &mockHttpClient{} _, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client) - assert.Equal(t, "a test must have at least one assertion, but none were found in 'testdata/tests/system-test/foo/query.json'\n", errBytes) + assert.Equal(t, "a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\n", errBytes) } func TestSuiteWithoutTests(t *testing.T) { client := &mockHttpClient{} outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) - assert.Equal(t, "Failed to find any tests at 'testdata/tests/staging-test'\n", outBytes) + assert.Equal(t, "Failed to find any tests at testdata/tests/staging-test\n", outBytes) assert.Equal(t, "", errBytes) } diff --git a/client/go/cmd/testdata/tests/expected-suite.out b/client/go/cmd/testdata/tests/expected-suite.out index 0fb8b897f4f..1830b879353 100644 --- a/client/go/cmd/testdata/tests/expected-suite.out +++ b/client/go/cmd/testdata/tests/expected-suite.out @@ -1,7 +1,9 @@ -Running testdata/tests/system-test/test.json: .... OK! -Running testdata/tests/system-test/wrong-bool-value.json: -Failed verifying assertion 0: -Expected JSON at /root/coverage/full (false) does not match actual (true). Response body: +Running my test: .... OK + +Running testdata/tests/system-test/wrong-bool-value.json: Failed step 1: +Unexpected value at /root/coverage/full: true +Expected: false +Actual response: { "root": { "children": [ @@ -36,9 +38,11 @@ Expected JSON at /root/coverage/full (false) does not match actual (true). Respo "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-element-count.json: -Failed verifying assertion 0: -Expected number of elements at /root/children (0) does not match actual (1). Response body: + +Running testdata/tests/system-test/wrong-element-count.json: Failed step 1: +Unexpected number of elements at /root/children: 1 +Expected: 0 +Actual response: { "root": { "children": [ @@ -73,9 +77,10 @@ Expected number of elements at /root/children (0) does not match actual (1). Res "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-field-name.json: -Failed verifying assertion 0: -Expected field at /root/fields/totalCountDracula not present in actual data. Response body: + +Running testdata/tests/system-test/wrong-field-name.json: Failed step 1: +Missing expected field at /root/fields/totalCountDracula +Actual response: { "root": { "children": [ @@ -110,9 +115,11 @@ Expected field at /root/fields/totalCountDracula not present in actual data. Res "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-float-value.json: -Failed verifying assertion 0: -Expected JSON at /root/children/0/relevance (0.381862373599) does not match actual (0.38186238359951247). Response body: + +Running testdata/tests/system-test/wrong-float-value.json: Failed step 1: +Unexpected value at /root/children/0/relevance: 0.38186238359951247 +Expected: 0.381862373599 +Actual response: { "root": { "children": [ @@ -147,9 +154,11 @@ Expected JSON at /root/children/0/relevance (0.381862373599) does not match actu "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-int-value.json: -Failed verifying assertion 0: -Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response body: + +Running testdata/tests/system-test/wrong-int-value.json: Failed step 1: +Unexpected value at /root/fields/totalCount: 1 +Expected: 2 +Actual response: { "root": { "children": [ @@ -184,9 +193,10 @@ Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-null-value.json: -Failed verifying assertion 0: -Expected field at /boot not present in actual data. Response body: + +Running testdata/tests/system-test/wrong-null-value.json: Failed step 1: +Missing expected field at /boot +Actual response: { "root": { "children": [ @@ -221,9 +231,11 @@ Expected field at /boot not present in actual data. Response body: "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-string-value.json: -Failed verifying assertion 0: -Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match actual ("Foo Fighters"). Response body: + +Running testdata/tests/system-test/wrong-string-value.json: Failed step 1: +Unexpected value at /root/children/0/fields/artist: "Foo Fighters" +Expected: "Boo Fighters" +Actual response: { "root": { "children": [ @@ -259,11 +271,52 @@ Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match } } -Failed 7 of 8 tests: -testdata/tests/system-test/wrong-bool-value.json: assertion 0 -testdata/tests/system-test/wrong-element-count.json: assertion 0 -testdata/tests/system-test/wrong-field-name.json: assertion 0 -testdata/tests/system-test/wrong-float-value.json: assertion 0 -testdata/tests/system-test/wrong-int-value.json: assertion 0 -testdata/tests/system-test/wrong-null-value.json: assertion 0 -testdata/tests/system-test/wrong-string-value.json: assertion 0 +Running testdata/tests/system-test/wrong-type.json: Failed step 1: +Unexpected type at /root/fields/totalCount: 1 +Expected: "1" +Actual response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + + +Failed 8 of 9 tests: +testdata/tests/system-test/wrong-bool-value.json: step 1: Unexpected value at /root/coverage/full: true +testdata/tests/system-test/wrong-element-count.json: step 1: Unexpected number of elements at /root/children: 1 +testdata/tests/system-test/wrong-field-name.json: step 1: Missing expected field at /root/fields/totalCountDracula +testdata/tests/system-test/wrong-float-value.json: step 1: Unexpected value at /root/children/0/relevance: 0.38186238359951247 +testdata/tests/system-test/wrong-int-value.json: step 1: Unexpected value at /root/fields/totalCount: 1 +testdata/tests/system-test/wrong-null-value.json: step 1: Missing expected field at /boot +testdata/tests/system-test/wrong-string-value.json: step 1: Unexpected value at /root/children/0/fields/artist: "Foo Fighters" +testdata/tests/system-test/wrong-type.json: step 1: Unexpected type at /root/fields/totalCount: 1 diff --git a/client/go/cmd/testdata/tests/expected.out b/client/go/cmd/testdata/tests/expected.out index f012ee30e95..b41f001cd25 100644 --- a/client/go/cmd/testdata/tests/expected.out +++ b/client/go/cmd/testdata/tests/expected.out @@ -1,2 +1,3 @@ -Running testdata/tests/system-test/test.json: .... OK! +Running my test: .... OK + 1 tests completed successfully diff --git a/client/go/cmd/testdata/tests/system-test/test.json b/client/go/cmd/testdata/tests/system-test/test.json index 5aac76d29ff..f53df929dbd 100644 --- a/client/go/cmd/testdata/tests/system-test/test.json +++ b/client/go/cmd/testdata/tests/system-test/test.json @@ -1,11 +1,12 @@ { + "name": "my test", "defaults": { "cluster": "container", "parameters": { "timeout": "3.4s" } }, - "assertions": [ + "steps": [ { "name": "feed music", "request": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json index ae6f9de8de8..c594a206347 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json index 77c687fa919..a772af67a78 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json index d020141ed12..6ce3d055584 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json index 804f2582176..6a1b221a91a 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json index 3cbf8acd1d8..d61a8b002c2 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json index 11425df7ad4..ea78357c99e 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json index 2cf0a5fdb38..5f56ebaab6d 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-type.json b/client/go/cmd/testdata/tests/system-test/wrong-type.json new file mode 100644 index 00000000000..6be28ff68ff --- /dev/null +++ b/client/go/cmd/testdata/tests/system-test/wrong-type.json @@ -0,0 +1,15 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "fields": { + "totalCount" : "1" + } + } + } + } + } + ] +} -- cgit v1.2.3 From e8e51d23c43f6e43ba914b9aa02ad09724d6c040 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 24 Nov 2021 17:22:19 +0100 Subject: No empty lines after successes --- client/go/cmd/test.go | 12 +++++++++--- client/go/cmd/testdata/tests/expected-suite.out | 2 -- 2 files changed, 9 insertions(+), 5 deletions(-) (limited to 'client') diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index fa2e47ce5d3..b5399bd213a 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -56,7 +56,7 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, fmt.Fprintf(stdout, "Failed to find any tests at %v\n", testPath) exitFunc(3) } else { - fmt.Fprintf(stdout, "%d tests completed successfully\n", count) + fmt.Fprintf(stdout, "\n%d tests completed successfully\n", count) } }, } @@ -71,12 +71,18 @@ func runTests(rootPath string, target vespa.Target) (int, []string) { if err != nil { fatalErr(err, "Failed reading specified test directory") } + previousFailed := false for _, test := range tests { if !test.IsDir() && filepath.Ext(test.Name()) == ".json" { testPath := path.Join(rootPath, test.Name()) + if previousFailed { + fmt.Fprintln(stdout, "") + previousFailed = false + } failure := runTest(testPath, target) if failure != "" { failed = append(failed, failure) + previousFailed = true } count++ } @@ -126,7 +132,7 @@ func runTest(testPath string, target vespa.Target) string { fatalErr(err, fmt.Sprintf("Error in %s", stepName)) } if failure != "" { - fmt.Fprintf(stdout, " Failed %s:\n%s\n\n", stepName, longFailure) + fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure) return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) } if i == 0 { @@ -134,7 +140,7 @@ func runTest(testPath string, target vespa.Target) string { } fmt.Fprint(stdout, ".") } - fmt.Fprintf(stdout, " OK\n\n") + fmt.Fprintln(stdout, " OK") return "" } diff --git a/client/go/cmd/testdata/tests/expected-suite.out b/client/go/cmd/testdata/tests/expected-suite.out index 1830b879353..f214f3cf160 100644 --- a/client/go/cmd/testdata/tests/expected-suite.out +++ b/client/go/cmd/testdata/tests/expected-suite.out @@ -1,5 +1,4 @@ Running my test: .... OK - Running testdata/tests/system-test/wrong-bool-value.json: Failed step 1: Unexpected value at /root/coverage/full: true Expected: false @@ -310,7 +309,6 @@ Actual response: } } - Failed 8 of 9 tests: testdata/tests/system-test/wrong-bool-value.json: step 1: Unexpected value at /root/coverage/full: true testdata/tests/system-test/wrong-element-count.json: step 1: Unexpected number of elements at /root/children: 1 -- cgit v1.2.3