Merge pull request #20190 from vespa-engine/jonmv/reapply-vespa-cli-test

Format changes, and rename to steps array

13 files changed, 183 insertions, 99 deletions

diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go
index dc75ca22729..b5399bd213a 100644
--- a/client/go/cmd/test.go
+++ b/client/go/cmd/test.go
@@ -53,10 +53,10 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`,
 			}
 			exitFunc(3)
 		} else if count == 0 {
-			fmt.Fprintf(stdout, "Failed to find any tests at '%v'\n", testPath)
+			fmt.Fprintf(stdout, "Failed to find any tests at %v\n", testPath)
 			exitFunc(3)
 		} else {
-			fmt.Fprintf(stdout, "%d tests completed successfully\n", count)
+			fmt.Fprintf(stdout, "\n%d tests completed successfully\n", count)
 		}
 	},
 }
@@ -71,12 +71,18 @@ func runTests(rootPath string, target vespa.Target) (int, []string) {
 		if err != nil {
 			fatalErr(err, "Failed reading specified test directory")
 		}
+		previousFailed := false
 		for _, test := range tests {
 			if !test.IsDir() && filepath.Ext(test.Name()) == ".json" {
 				testPath := path.Join(rootPath, test.Name())
+				if previousFailed {
+					fmt.Fprintln(stdout, "")
+					previousFailed = false
+				}
 				failure := runTest(testPath, target)
 				if failure != "" {
 					failed = append(failed, failure)
+					previousFailed = true
 				}
 				count++
 			}
@@ -96,10 +102,10 @@ func runTest(testPath string, target vespa.Target) string {
 	var test test
 	testBytes, err := ioutil.ReadFile(testPath)
 	if err != nil {
-		fatalErr(err, fmt.Sprintf("Failed to read test file at '%s'", testPath))
+		fatalErr(err, fmt.Sprintf("Failed to read test file at %s", testPath))
 	}
 	if err = json.Unmarshal(testBytes, &test); err != nil {
-		fatalErr(err, fmt.Sprintf("Failed to parse test file at '%s", testPath))
+		fatalErr(err, fmt.Sprintf("Failed to parse test file at %s", testPath))
 	}
 
 	testName := test.Name
@@ -110,44 +116,44 @@ func runTest(testPath string, target vespa.Target) string {
 
 	defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath))
 	if err != nil {
-		fatalErr(err, fmt.Sprintf("Invalid default parameters for '%s'", testName))
+		fatalErr(err, fmt.Sprintf("Invalid default parameters for %s", testName))
 	}
 
-	if len(test.Assertions) == 0 {
-		fatalErr(fmt.Errorf("a test must have at least one assertion, but none were found in '%s'", testPath))
+	if len(test.Steps) == 0 {
+		fatalErr(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath))
 	}
-	for i, assertion := range test.Assertions {
-		assertionName := assertion.Name
-		if assertionName == "" {
-			assertionName = fmt.Sprintf("assertion %d", i)
+	for i, step := range test.Steps {
+		stepName := step.Name
+		if stepName == "" {
+			stepName = fmt.Sprintf("step %d", i + 1)
 		}
-		failure, err := verify(assertion, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target)
+		failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target)
 		if err != nil {
-			fatalErr(err, fmt.Sprintf("\nError verifying %s", assertionName))
+			fatalErr(err, fmt.Sprintf("Error in %s", stepName))
 		}
 		if failure != "" {
-			fmt.Fprintf(stdout, "\nFailed verifying %s:\n%s\n", assertionName, failure)
-			return fmt.Sprintf("%v: %v", testName, assertionName)
+			fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure)
+			return fmt.Sprintf("%s: %s: %s", testName, stepName, failure)
 		}
 		if i == 0 {
 			fmt.Fprintf(stdout, " ")
 		}
 		fmt.Fprint(stdout, ".")
 	}
-	fmt.Fprintln(stdout, " OK!")
+	fmt.Fprintln(stdout, " OK")
 	return ""
 }
 
 // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails
-func verify(assertion assertion, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, error) {
-	requestBody, err := getBody(assertion.Request.BodyRaw, testsPath)
+func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, string, error) {
+	requestBody, err := getBody(step.Request.BodyRaw, testsPath)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
-	parameters, err := getParameters(assertion.Request.ParametersRaw, testsPath)
+	parameters, err := getParameters(step.Request.ParametersRaw, testsPath)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	for name, value := range defaultParameters {
 		if _, present := parameters[name]; !present {
@@ -155,28 +161,28 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul
 		}
 	}
 
-	cluster := assertion.Request.Cluster
+	cluster := step.Request.Cluster
 	if cluster == "" {
 		cluster = defaultCluster
 	}
 
 	service, err := target.Service("query", 0, 0, cluster)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
-	method := assertion.Request.Method
+	method := step.Request.Method
 	if method == "" {
 		method = "GET"
 	}
 
-	pathAndQuery := assertion.Request.URI
+	pathAndQuery := step.Request.URI
 	if pathAndQuery == "" {
 		pathAndQuery = "/search/"
 	}
 	requestUrl, err := url.ParseRequestURI(service.BaseURL + pathAndQuery)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	query := requestUrl.Query()
 	for name, value := range parameters {
@@ -197,50 +203,56 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul
 
 	response, err := service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	defer response.Body.Close()
 
-	statusCode := assertion.Response.Code
+	statusCode := step.Response.Code
 	if statusCode == 0 {
 		statusCode = 200
 	}
 	if statusCode != response.StatusCode {
-		return fmt.Sprintf("Expected status code (%d) does not match actual (%d). Response body:\n%s", statusCode, response.StatusCode, util.ReaderToJSON(response.Body)), nil
+		failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode)
+		return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil
 	}
 
-	responseBodySpecBytes, err := getBody(assertion.Response.BodyRaw, testsPath)
+	responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	if responseBodySpecBytes == nil {
-		return "", nil
+		return "", "", nil
 	}
 	var responseBodySpec interface{}
 	err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
 	responseBodyBytes, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	var responseBody interface{}
 	err = json.Unmarshal(responseBodyBytes, &responseBody)
 	if err != nil {
-		return "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes))
+		return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes))
 	}
 
-	failure, err := compare(responseBodySpec, responseBody, "")
+	failure, expected, err := compare(responseBodySpec, responseBody, "")
 	if failure != "" {
 		responsePretty, _ := json.MarshalIndent(responseBody, "", "  ")
-		failure = failure + " Response body:\n" + string(responsePretty)
+		longFailure := failure
+		if expected != "" {
+			longFailure += "\n" + expected
+		}
+		longFailure += "\nActual response:\n" + string(responsePretty)
+		return failure, longFailure, err
 	}
-	return failure, err
+	return "", "", err
 }
 
-func compare(expected interface{}, actual interface{}, path string) (string, error) {
+func compare(expected interface{}, actual interface{}, path string) (string, string, error) {
 	typeMatch := false
 	valueMatch := false
 	switch u := expected.(type) {
@@ -265,14 +277,14 @@ func compare(expected interface{}, actual interface{}, path string) (string, err
 		if ok {
 			if len(u) == len(v) {
 				for i, e := range u {
-					result, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i))
-					if result != "" || err != nil {
-						return result, err
+					failure, expected, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i))
+					if failure != "" || err != nil {
+						return failure, expected, err
 					}
 				}
 				valueMatch = true
 			} else {
-				return fmt.Sprintf("Expected number of elements at %s (%d) does not match actual (%d).", path, len(u), len(v)), nil
+				return fmt.Sprintf("Unexpected number of elements at %s: %d", path, len(v)), fmt.Sprintf("Expected: %d", len(u)), nil
 			}
 		}
 	case map[string]interface{}:
@@ -283,28 +295,32 @@ func compare(expected interface{}, actual interface{}, path string) (string, err
 				childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1"))
 				f, ok := v[n]
 				if !ok {
-					return fmt.Sprintf("Expected field at %s not present in actual data.", childPath), nil
+					return fmt.Sprintf("Missing expected field at %s", childPath), "", nil
 				}
-				result, err := compare(e, f, childPath)
-				if result != "" || err != nil {
-					return result, err
+				failure, expected, err := compare(e, f, childPath)
+				if failure != "" || err != nil {
+					return failure, expected, err
 				}
 			}
 			valueMatch = true
 		}
 	default:
-		return "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected)
+		return "", "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected)
 	}
 
-	if !(typeMatch && valueMatch) {
+	if !valueMatch {
 		if path == "" {
 			path = "root"
 		}
-		expectedJson, _ := json.MarshalIndent(expected, "", "  ")
-		actualJson, _ := json.MarshalIndent(actual, "", "  ")
-		return fmt.Sprintf("Expected JSON at %s (%s) does not match actual (%s).", path, expectedJson, actualJson), nil
+		mismatched := "type"
+		if typeMatch {
+			mismatched = "value"
+		}
+		expectedJson, _ := json.Marshal(expected)
+		actualJson, _ := json.Marshal(actual)
+		return fmt.Sprintf("Unexpected %s at %s: %s", mismatched, path, actualJson), fmt.Sprintf("Expected: %s", expectedJson), nil
 	}
-	return "", nil
+	return "", "", nil
 }
 
 func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) {
@@ -340,8 +356,8 @@ func getBody(bodyRaw []byte, testsPath string) ([]byte, error) {
 
 type test struct {
 	Name       string      `json:"name"`
-	Defaults   defaults    `json:"defaults"`
-	Assertions []assertion `json:"assertions"`
+	Defaults defaults `json:"defaults"`
+	Steps    []step   `json:"steps"`
 }
 
 type defaults struct {
@@ -349,7 +365,7 @@ type defaults struct {
 	ParametersRaw json.RawMessage `json:"parameters"`
 }
 
-type assertion struct {
+type step struct {
 	Name     string   `json:"name"`
 	Request  request  `json:"request"`
 	Response response `json:"response"`
diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go
index 9d92e285750..8db10282d51 100644
--- a/client/go/cmd/test_test.go
+++ b/client/go/cmd/test_test.go
@@ -23,7 +23,7 @@ func TestSuite(t *testing.T) {
 	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
 	client.NextStatus(200)
 	client.NextStatus(200)
-	for i := 0; i < 9; i++ {
+	for i := 0; i < 10; i++ {
 		client.NextResponse(200, string(searchResponse))
 	}
 
@@ -35,7 +35,7 @@ func TestSuite(t *testing.T) {
 	baseUrl := "http://127.0.0.1:8080"
 	urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
 	requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)}
-	for i := 0; i < 7; i++ {
+	for i := 0; i < 8; i++ {
 		requests = append(requests, createSearchRequest(baseUrl+"/search/"))
 	}
 	assertRequests(requests, client, t)
@@ -44,13 +44,13 @@ func TestSuite(t *testing.T) {
 func TestTestWithoutAssertions(t *testing.T) {
 	client := &mockHttpClient{}
 	_, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client)
-	assert.Equal(t, "a test must have at least one assertion, but none were found in 'testdata/tests/system-test/foo/query.json'\n", errBytes)
+	assert.Equal(t, "a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\n", errBytes)
 }
 
 func TestSuiteWithoutTests(t *testing.T) {
 	client := &mockHttpClient{}
 	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client)
-	assert.Equal(t, "Failed to find any tests at 'testdata/tests/staging-test'\n", outBytes)
+	assert.Equal(t, "Failed to find any tests at testdata/tests/staging-test\n", outBytes)
 	assert.Equal(t, "", errBytes)
 }
 
diff --git a/client/go/cmd/testdata/tests/expected-suite.out b/client/go/cmd/testdata/tests/expected-suite.out
index 0fb8b897f4f..f214f3cf160 100644
--- a/client/go/cmd/testdata/tests/expected-suite.out
+++ b/client/go/cmd/testdata/tests/expected-suite.out
@@ -1,7 +1,8 @@
-Running testdata/tests/system-test/test.json: .... OK!
-Running testdata/tests/system-test/wrong-bool-value.json:
-Failed verifying assertion 0:
-Expected JSON at /root/coverage/full (false) does not match actual (true). Response body:
+Running my test: .... OK
+Running testdata/tests/system-test/wrong-bool-value.json: Failed step 1:
+Unexpected value at /root/coverage/full: true
+Expected: false
+Actual response:
 {
   "root": {
     "children": [
@@ -36,9 +37,11 @@ Expected JSON at /root/coverage/full (false) does not match actual (true). Respo
     "summaryfetchtime": 0
   }
 }
-Running testdata/tests/system-test/wrong-element-count.json:
-Failed verifying assertion 0:
-Expected number of elements at /root/children (0) does not match actual (1). Response body:
+
+Running testdata/tests/system-test/wrong-element-count.json: Failed step 1:
+Unexpected number of elements at /root/children: 1
+Expected: 0
+Actual response:
 {
   "root": {
     "children": [
@@ -73,9 +76,10 @@ Expected number of elements at /root/children (0) does not match actual (1). Res
     "summaryfetchtime": 0
   }
 }
-Running testdata/tests/system-test/wrong-field-name.json:
-Failed verifying assertion 0:
-Expected field at /root/fields/totalCountDracula not present in actual data. Response body:
+
+Running testdata/tests/system-test/wrong-field-name.json: Failed step 1:
+Missing expected field at /root/fields/totalCountDracula
+Actual response:
 {
   "root": {
     "children": [
@@ -110,9 +114,11 @@ Expected field at /root/fields/totalCountDracula not present in actual data. Res
     "summaryfetchtime": 0
   }
 }
-Running testdata/tests/system-test/wrong-float-value.json:
-Failed verifying assertion 0:
-Expected JSON at /root/children/0/relevance (0.381862373599) does not match actual (0.38186238359951247). Response body:
+
+Running testdata/tests/system-test/wrong-float-value.json: Failed step 1:
+Unexpected value at /root/children/0/relevance: 0.38186238359951247
+Expected: 0.381862373599
+Actual response:
 {
   "root": {
     "children": [
@@ -147,9 +153,11 @@ Expected JSON at /root/children/0/relevance (0.381862373599) does not match actu
     "summaryfetchtime": 0
   }
 }
-Running testdata/tests/system-test/wrong-int-value.json:
-Failed verifying assertion 0:
-Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response body:
+
+Running testdata/tests/system-test/wrong-int-value.json: Failed step 1:
+Unexpected value at /root/fields/totalCount: 1
+Expected: 2
+Actual response:
 {
   "root": {
     "children": [
@@ -184,9 +192,10 @@ Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response
     "summaryfetchtime": 0
   }
 }
-Running testdata/tests/system-test/wrong-null-value.json:
-Failed verifying assertion 0:
-Expected field at /boot not present in actual data. Response body:
+
+Running testdata/tests/system-test/wrong-null-value.json: Failed step 1:
+Missing expected field at /boot
+Actual response:
 {
   "root": {
     "children": [
@@ -221,9 +230,50 @@ Expected field at /boot not present in actual data. Response body:
     "summaryfetchtime": 0
   }
 }
-Running testdata/tests/system-test/wrong-string-value.json:
-Failed verifying assertion 0:
-Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match actual ("Foo Fighters"). Response body:
+
+Running testdata/tests/system-test/wrong-string-value.json: Failed step 1:
+Unexpected value at /root/children/0/fields/artist: "Foo Fighters"
+Expected: "Boo Fighters"
+Actual response:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+
+Running testdata/tests/system-test/wrong-type.json: Failed step 1:
+Unexpected type at /root/fields/totalCount: 1
+Expected: "1"
+Actual response:
 {
   "root": {
     "children": [
@@ -259,11 +309,12 @@ Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match
   }
 }
 
-Failed 7 of 8 tests:
-testdata/tests/system-test/wrong-bool-value.json: assertion 0
-testdata/tests/system-test/wrong-element-count.json: assertion 0
-testdata/tests/system-test/wrong-field-name.json: assertion 0
-testdata/tests/system-test/wrong-float-value.json: assertion 0
-testdata/tests/system-test/wrong-int-value.json: assertion 0
-testdata/tests/system-test/wrong-null-value.json: assertion 0
-testdata/tests/system-test/wrong-string-value.json: assertion 0
+Failed 8 of 9 tests:
+testdata/tests/system-test/wrong-bool-value.json: step 1: Unexpected value at /root/coverage/full: true
+testdata/tests/system-test/wrong-element-count.json: step 1: Unexpected number of elements at /root/children: 1
+testdata/tests/system-test/wrong-field-name.json: step 1: Missing expected field at /root/fields/totalCountDracula
+testdata/tests/system-test/wrong-float-value.json: step 1: Unexpected value at /root/children/0/relevance: 0.38186238359951247
+testdata/tests/system-test/wrong-int-value.json: step 1: Unexpected value at /root/fields/totalCount: 1
+testdata/tests/system-test/wrong-null-value.json: step 1: Missing expected field at /boot
+testdata/tests/system-test/wrong-string-value.json: step 1: Unexpected value at /root/children/0/fields/artist: "Foo Fighters"
+testdata/tests/system-test/wrong-type.json: step 1: Unexpected type at /root/fields/totalCount: 1
diff --git a/client/go/cmd/testdata/tests/expected.out b/client/go/cmd/testdata/tests/expected.out
index f012ee30e95..b41f001cd25 100644
--- a/client/go/cmd/testdata/tests/expected.out
+++ b/client/go/cmd/testdata/tests/expected.out
@@ -1,2 +1,3 @@
-Running testdata/tests/system-test/test.json: .... OK!
+Running my test: .... OK
+
 1 tests completed successfully
diff --git a/client/go/cmd/testdata/tests/system-test/test.json b/client/go/cmd/testdata/tests/system-test/test.json
index 5aac76d29ff..f53df929dbd 100644
--- a/client/go/cmd/testdata/tests/system-test/test.json
+++ b/client/go/cmd/testdata/tests/system-test/test.json
@@ -1,11 +1,12 @@
 {
+    "name": "my test",
     "defaults": {
         "cluster": "container",
         "parameters": {
             "timeout": "3.4s"
         }
     },
-    "assertions": [
+    "steps": [
         {
             "name": "feed music",
             "request": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json
index ae6f9de8de8..c594a206347 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json
index 77c687fa919..a772af67a78 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json
index d020141ed12..6ce3d055584 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json
index 804f2582176..6a1b221a91a 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json
index 3cbf8acd1d8..d61a8b002c2 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json
index 11425df7ad4..ea78357c99e 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json
index 2cf0a5fdb38..5f56ebaab6d 100644
--- a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json
+++ b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json
@@ -1,5 +1,5 @@
 {
-    "assertions": [
+    "steps": [
         {
             "response": {
                 "body": {
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-type.json b/client/go/cmd/testdata/tests/system-test/wrong-type.json
new file mode 100644
index 00000000000..6be28ff68ff
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-type.json
@@ -0,0 +1,15 @@
+{
+    "steps": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "fields": {
+                            "totalCount" : "1"
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}