Unit tests for test command

17 files changed, 631 insertions, 4 deletions

diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go
index 4519b1c1e0e..4c6d9ec265d 100644
--- a/client/go/cmd/test.go
+++ b/client/go/cmd/test.go
@@ -106,7 +106,7 @@ func runTest(testPath string, target vespa.Target) string {
 	if test.Name == "" {
 		testName = testPath
 	}
-	fmt.Fprintf(stdout, "Running %s: ", testName)
+	fmt.Fprintf(stdout, "Running %s:", testName)
 
 	defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath))
 	if err != nil {
@@ -126,12 +126,15 @@ func runTest(testPath string, target vespa.Target) string {
 			fatalErr(err, fmt.Sprintf("\nError verifying %s", assertionName))
 		}
 		if failure != "" {
-			fmt.Fprintf(stdout, "\nFailed verifying %s: \n%s\n", assertionName, failure)
+			fmt.Fprintf(stdout, "\nFailed verifying %s:\n%s\n", assertionName, failure)
 			return fmt.Sprintf("%v: %v", testName, assertionName)
 		}
-		fmt.Print(".")
+		if i == 0 {
+			fmt.Fprintf(stdout, " ")
+		}
+		fmt.Fprint(stdout, ".")
 	}
-	fmt.Println(" OK!")
+	fmt.Fprintln(stdout, " OK!")
 	return ""
 }
 
diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go
new file mode 100644
index 00000000000..73ea13331c7
--- /dev/null
+++ b/client/go/cmd/test_test.go
@@ -0,0 +1,129 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// test command tests
+// Author: jonmv
+
+package cmd
+
+import (
+	"github.com/vespa-engine/vespa/client/go/util"
+	"github.com/vespa-engine/vespa/client/go/vespa"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSuite(t *testing.T) {
+	client := &mockHttpClient{}
+	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
+	client.NextStatus(200)
+	client.NextStatus(200)
+	for i := 0; i < 9; i++ {
+		client.NextResponse(200, string(searchResponse))
+	}
+
+	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected-suite.out")
+	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test"}}, t, client)
+	assert.Equal(t, string(expectedBytes), outBytes)
+	assert.Equal(t, "", errBytes)
+
+	baseUrl := "http://127.0.0.1:8080"
+	urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
+	requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)}
+	for i := 0; i < 7; i++ {
+		requests = append(requests, createSearchRequest(baseUrl+"/search/"))
+	}
+	assertRequests(requests, client, t)
+}
+
+func TestTestWithoutAssertions(t *testing.T) {
+	client := &mockHttpClient{}
+	_, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client)
+	assert.Equal(t, "a test must have at least one assertion, but none were found in 'testdata/tests/system-test/foo/query.json'\n", errBytes)
+}
+
+func TestSingleTest(t *testing.T) {
+	client := &mockHttpClient{}
+	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
+	client.NextStatus(200)
+	client.NextStatus(200)
+	client.NextResponse(200, string(searchResponse))
+	client.NextResponse(200, string(searchResponse))
+
+	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out")
+	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/test.json"}}, t, client)
+	assert.Equal(t, string(expectedBytes), outBytes)
+	assert.Equal(t, "", errBytes)
+
+	baseUrl := "http://127.0.0.1:8080"
+	rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
+	assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t)
+}
+
+func TestSingleTestWithCloudAndEndpoints(t *testing.T) {
+	cmd := command{args: []string{"test", "testdata/tests/system-test/test.json", "-t", "cloud", "-a", "t.a.i"}}
+	cmd.homeDir = filepath.Join(t.TempDir(), ".vespa")
+	os.MkdirAll(cmd.homeDir, 0700)
+	keyFile := filepath.Join(cmd.homeDir, "key")
+	certFile := filepath.Join(cmd.homeDir, "cert")
+
+	os.Setenv("VESPA_CLI_DATA_PLANE_KEY_FILE", keyFile)
+	os.Setenv("VESPA_CLI_DATA_PLANE_CERT_FILE", certFile)
+	os.Setenv("VESPA_CLI_ENDPOINTS", "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"https://url\"}]}")
+
+	kp, _ := vespa.CreateKeyPair()
+	ioutil.WriteFile(keyFile, kp.PrivateKey, 0600)
+	ioutil.WriteFile(certFile, kp.Certificate, 0600)
+
+	client := &mockHttpClient{}
+	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
+	client.NextStatus(200)
+	client.NextStatus(200)
+	client.NextResponse(200, string(searchResponse))
+	client.NextResponse(200, string(searchResponse))
+
+	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out")
+	outBytes, errBytes := execute(cmd, t, client)
+	assert.Equal(t, string(expectedBytes), outBytes)
+	assert.Equal(t, "", errBytes)
+
+	baseUrl := "https://url"
+	rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
+	assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t)
+}
+
+func createFeedRequest(urlPrefix string) *http.Request {
+	return createRequest("POST",
+		urlPrefix+"/document/v1/test/music/docid/doc?timeout=3.4s",
+		"{\"fields\":{\"artist\":\"Foo Fighters\"}}")
+}
+
+func createSearchRequest(rawUrl string) *http.Request {
+	return createRequest("GET", rawUrl, "")
+}
+
+func createRequest(method string, uri string, body string) *http.Request {
+	requestUrl, _ := url.ParseRequestURI(uri)
+	return &http.Request{
+		URL:    requestUrl,
+		Method: method,
+		Header: nil,
+		Body:   ioutil.NopCloser(strings.NewReader(body)),
+	}
+}
+
+func assertRequests(requests []*http.Request, client *mockHttpClient, t *testing.T) {
+	if assert.Equal(t, len(requests), len(client.requests)) {
+		for i, e := range requests {
+			a := client.requests[i]
+			assert.Equal(t, e.URL.String(), a.URL.String())
+			assert.Equal(t, e.Method, a.Method)
+			assert.Equal(t, util.ReaderToJSON(e.Body), util.ReaderToJSON(a.Body))
+		}
+	}
+}
diff --git a/client/go/cmd/testdata/tests/body.json b/client/go/cmd/testdata/tests/body.json
new file mode 100644
index 00000000000..767330b1a2d
--- /dev/null
+++ b/client/go/cmd/testdata/tests/body.json
@@ -0,0 +1,12 @@
+{
+  "root": {
+    "id": "toplevel",
+    "coverage": {
+      "full": true
+    },
+    "fields": {
+      "totalCount" : 1
+    },
+    "children": [{}]
+  }
+}
\ No newline at end of file
diff --git a/client/go/cmd/testdata/tests/expected-suite.out b/client/go/cmd/testdata/tests/expected-suite.out
new file mode 100644
index 00000000000..0fb8b897f4f
--- /dev/null
+++ b/client/go/cmd/testdata/tests/expected-suite.out
@@ -0,0 +1,269 @@
+Running testdata/tests/system-test/test.json: .... OK!
+Running testdata/tests/system-test/wrong-bool-value.json:
+Failed verifying assertion 0:
+Expected JSON at /root/coverage/full (false) does not match actual (true). Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+Running testdata/tests/system-test/wrong-element-count.json:
+Failed verifying assertion 0:
+Expected number of elements at /root/children (0) does not match actual (1). Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+Running testdata/tests/system-test/wrong-field-name.json:
+Failed verifying assertion 0:
+Expected field at /root/fields/totalCountDracula not present in actual data. Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+Running testdata/tests/system-test/wrong-float-value.json:
+Failed verifying assertion 0:
+Expected JSON at /root/children/0/relevance (0.381862373599) does not match actual (0.38186238359951247). Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+Running testdata/tests/system-test/wrong-int-value.json:
+Failed verifying assertion 0:
+Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+Running testdata/tests/system-test/wrong-null-value.json:
+Failed verifying assertion 0:
+Expected field at /boot not present in actual data. Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+Running testdata/tests/system-test/wrong-string-value.json:
+Failed verifying assertion 0:
+Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match actual ("Foo Fighters"). Response body:
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
+
+Failed 7 of 8 tests:
+testdata/tests/system-test/wrong-bool-value.json: assertion 0
+testdata/tests/system-test/wrong-element-count.json: assertion 0
+testdata/tests/system-test/wrong-field-name.json: assertion 0
+testdata/tests/system-test/wrong-float-value.json: assertion 0
+testdata/tests/system-test/wrong-int-value.json: assertion 0
+testdata/tests/system-test/wrong-null-value.json: assertion 0
+testdata/tests/system-test/wrong-string-value.json: assertion 0
diff --git a/client/go/cmd/testdata/tests/expected.out b/client/go/cmd/testdata/tests/expected.out
new file mode 100644
index 00000000000..f012ee30e95
--- /dev/null
+++ b/client/go/cmd/testdata/tests/expected.out
@@ -0,0 +1,2 @@
+Running testdata/tests/system-test/test.json: .... OK!
+1 tests completed successfully
diff --git a/client/go/cmd/testdata/tests/response.json b/client/go/cmd/testdata/tests/response.json
index e69de29bb2d..48368b935a8 100644
--- a/client/go/cmd/testdata/tests/response.json
+++ b/client/go/cmd/testdata/tests/response.json
@@ -0,0 +1,34 @@
+{
+  "root": {
+    "children": [
+      {
+        "fields": {
+          "artist": "Foo Fighters",
+          "documentid": "id:test:music::doc",
+          "sddocname": "music"
+        },
+        "id": "id:test:music::doc",
+        "relevance": 0.38186238359951247,
+        "source": "music"
+      }
+    ],
+    "coverage": {
+      "coverage": 100,
+      "documents": 1,
+      "full": true,
+      "nodes": 1,
+      "results": 1,
+      "resultsFull": 1
+    },
+    "fields": {
+      "totalCount": 1
+    },
+    "id": "toplevel",
+    "relevance": 1
+  },
+  "timing": {
+    "querytime": 0.003,
+    "searchtime": 0.004,
+    "summaryfetchtime": 0
+  }
+}
\ No newline at end of file
diff --git a/client/go/cmd/testdata/tests/staging-test/not-json b/client/go/cmd/testdata/tests/staging-test/not-json
new file mode 100644
index 00000000000..b6fc4c620b6
--- /dev/null
+++ b/client/go/cmd/testdata/tests/staging-test/not-json
@@ -0,0 +1 @@
+hello
\ No newline at end of file
diff --git a/client/go/cmd/testdata/tests/system-test/foo/body.json b/client/go/cmd/testdata/tests/system-test/foo/body.json
new file mode 100644
index 00000000000..0bbf626eafe
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/foo/body.json
@@ -0,0 +1,5 @@
+{
+  "fields": {
+    "artist": "Foo Fighters"
+  }
+}
\ No newline at end of file
diff --git a/client/go/cmd/testdata/tests/system-test/foo/query.json b/client/go/cmd/testdata/tests/system-test/foo/query.json
new file mode 100644
index 00000000000..25b8c5b0039
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/foo/query.json
@@ -0,0 +1,3 @@
+{
+    "query": "artist: foo"
+}
diff --git a/client/go/cmd/testdata/tests/system-test/test.json b/client/go/cmd/testdata/tests/system-test/test.json
new file mode 100644
index 00000000000..5aac76d29ff
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/test.json
@@ -0,0 +1,64 @@
+{
+    "defaults": {
+        "cluster": "container",
+        "parameters": {
+            "timeout": "3.4s"
+        }
+    },
+    "assertions": [
+        {
+            "name": "feed music",
+            "request": {
+                "method": "POST",
+                "body": "foo/body.json",
+                "uri": "/document/v1/test/music/docid/doc"
+            }
+        },
+        {
+            "name": "re-feed music",
+            "request": {
+                "method": "POST",
+                "body": { 
+                    "fields": {
+                        "artist": "Foo Fighters"
+                    }
+                },
+                "uri": "/document/v1/test/music/docid/doc"
+            }
+        },
+        {
+            "name": "query for foo",
+            "request": {
+                "uri": "/search/?presentation.timing=true",
+                "parameters": {
+                    "query": "artist: foo"
+                }
+            },
+            "response": {
+                "code": 200,
+                "body": "../body.json"
+            }
+        },
+        {
+            "name": "query for foo again",
+            "request": {
+                "uri": "/search/?presentation.timing=true",
+                "parameters": "foo/query.json"
+            },
+            "response": {
+                "body": {
+                    "root": {
+                        "children": [
+                            {
+                                "fields": {
+                                    "artist": "Foo Fighters"
+                                },
+                                "relevance": 0.381862383599
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json
new file mode 100644
index 00000000000..ae6f9de8de8
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json
@@ -0,0 +1,15 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "coverage": {
+                            "full": false
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json
new file mode 100644
index 00000000000..77c687fa919
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json
@@ -0,0 +1,13 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "children": []
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json
new file mode 100644
index 00000000000..d020141ed12
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json
@@ -0,0 +1,15 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "fields": {
+                            "totalCountDracula" : 1
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json
new file mode 100644
index 00000000000..804f2582176
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json
@@ -0,0 +1,17 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "children": [
+                            {
+                                "relevance": 0.381862373599
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json
new file mode 100644
index 00000000000..3cbf8acd1d8
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json
@@ -0,0 +1,15 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "fields": {
+                            "totalCount" : 2
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json
new file mode 100644
index 00000000000..11425df7ad4
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json
@@ -0,0 +1,11 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "boot": null
+                }
+            }
+        }
+    ]
+}
diff --git a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json
new file mode 100644
index 00000000000..2cf0a5fdb38
--- /dev/null
+++ b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json
@@ -0,0 +1,19 @@
+{
+    "assertions": [
+        {
+            "response": {
+                "body": {
+                    "root": {
+                        "children": [
+                            {
+                                "fields": {
+                                    "artist": "Boo Fighters"
+                                }
+                            }
+                        ]
+                    }
+                }
+            }
+        }
+    ]
+}