diff options
Diffstat (limited to 'client')
-rw-r--r-- | client/go/cmd/prod.go | 20 | ||||
-rw-r--r-- | client/go/cmd/prod_test.go | 57 | ||||
-rw-r--r-- | client/go/cmd/test.go | 123 | ||||
-rw-r--r-- | client/go/cmd/test_test.go | 7 | ||||
-rw-r--r-- | client/go/vespa/deploy.go | 11 |
5 files changed, 153 insertions, 65 deletions
diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index d1d72362448..89dc4cb6094 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -116,7 +116,7 @@ For more information about production deployments in Vespa Cloud see: https://cloud.vespa.ai/en/getting-to-production https://cloud.vespa.ai/en/automated-deployments`, DisableAutoGenTag: true, - Example: `$ mvn package + Example: `$ mvn package # when adding custom Java components $ vespa prod submit`, Run: func(cmd *cobra.Command, args []string) { target := getTarget() @@ -139,10 +139,13 @@ $ vespa prod submit`, fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init") return } - if !pkg.IsJava() { - // TODO: Loosen this requirement when we start supporting applications with Java in production - fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported") + if pkg.TestPath == "" { + fatalErrHint(fmt.Errorf("No tests found"), + "The application must be a Java maven project, or include basic HTTP tests under src/test/application/", + "See https://cloud.vespa.ai/en/reference/getting-to-production") return + } else { + verifyTests(pkg.TestPath, target) } isCI := os.Getenv("CI") != "" if !isCI { @@ -347,3 +350,12 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu } return input } + +func verifyTests(testsParent string, target vespa.Target) { + runTests(filepath.Join(testsParent, "tests", "system-test"), target, true) + runTests(filepath.Join(testsParent, "tests", "staging-setup"), target, true) + runTests(filepath.Join(testsParent, "tests", "staging-test"), target, true) + if util.PathExists(filepath.Join(testsParent, "tests", "production-test")) { + runTests(filepath.Join(testsParent, "tests", "production-test"), target, true) + } +} diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 4ce6112122a..a4f3ebd6b56 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -16,7 +16,7 @@ import ( func TestProdInit(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") - createApplication(t, pkgDir) + createApplication(t, pkgDir, false) answers := []string{ // Regions @@ -81,7 +81,7 @@ func readFileString(t *testing.T, filename string) string { return string(content) } -func createApplication(t *testing.T, pkgDir string) { +func createApplication(t *testing.T, pkgDir string, java bool) { appDir := filepath.Join(pkgDir, "src", "main", "application") targetDir := filepath.Join(pkgDir, "target") if err := os.MkdirAll(appDir, 0755); err != nil { @@ -120,7 +120,24 @@ func createApplication(t *testing.T, pkgDir string) { if err := os.MkdirAll(targetDir, 0755); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + if java { + if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } else { + testsDir := filepath.Join(pkgDir, "src", "test", "application", "tests") + testBytes, _ := ioutil.ReadAll(strings.NewReader("{\"steps\":[{}]}")) + writeTest(filepath.Join(testsDir, "system-test", "test.json"), testBytes, t) + writeTest(filepath.Join(testsDir, "staging-setup", "test.json"), testBytes, t) + writeTest(filepath.Join(testsDir, "staging-test", "test.json"), testBytes, t) + } +} + +func writeTest(path string, content []byte, t *testing.T) { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path, content, 0644); err != nil { t.Fatal(err) } } @@ -128,7 +145,37 @@ func createApplication(t *testing.T, pkgDir string) { func TestProdSubmit(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") - createApplication(t, pkgDir) + createApplication(t, pkgDir, false) + + httpClient := &mockHttpClient{} + httpClient.NextResponse(200, `ok`) + execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + + // Zipping requires relative paths, so much let command run from pkgDir, then reset cwd for subsequent tests. + if cwd, err := os.Getwd(); err != nil { + t.Fatal(err) + } else { + defer os.Chdir(cwd) + } + if err := os.Chdir(pkgDir); err != nil { + t.Fatal(err) + } + if err := os.Setenv("CI", "true"); err != nil { + t.Fatal(err) + } + out, err := execute(command{homeDir: homeDir, args: []string{"prod", "submit"}}, t, httpClient) + assert.Equal(t, "", err) + assert.Contains(t, out, "Success: Submitted") + assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress") +} + +func TestProdSubmitWithJava(t *testing.T) { + homeDir := filepath.Join(t.TempDir(), ".vespa") + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, true) httpClient := &mockHttpClient{} httpClient.NextResponse(200, `ok`) @@ -137,7 +184,7 @@ func TestProdSubmit(t *testing.T) { execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) - // Copy an application package pre-assambled with mvn package + // Copy an application package pre-assembled with mvn package testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") zipFile := filepath.Join(testAppDir, "application.zip") copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile) diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index d6fdc9d2e41..7c49703595e 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -27,16 +27,15 @@ func init() { rootCmd.AddCommand(testCmd) } -// TODO: add link to test doc at cloud.vespa.ai var testCmd = &cobra.Command{ Use: "test [tests directory or test file]", Short: "Run a test suite, or a single test", Long: `Run a test suite, or a single test -Runs all JSON test files in the specified directory, or the single JSON -test file specified. +Runs all JSON test files in the specified directory (the working +directory by default), or the single JSON test file specified. -If no directory or file is specified, the working directory is used instead.`, +See https://cloud.vespa.ai/en/reference/testing.html for details.`, Example: `$ vespa test src/test/application/tests/system-test $ vespa test src/test/application/tests/system-test/feed-and-query.json`, Args: cobra.MaximumNArgs(1), @@ -47,15 +46,12 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, if len(args) > 0 { testPath = args[0] } - if count, failed := runTests(testPath, target); len(failed) != 0 { + if count, failed := runTests(testPath, target, false); len(failed) != 0 { fmt.Fprintf(stdout, "\nFailed %d of %d tests:\n", len(failed), count) for _, test := range failed { fmt.Fprintln(stdout, test) } exitFunc(3) - } else if count == 0 { - fmt.Fprintf(stdout, "Failed to find any tests at %v\n", testPath) - exitFunc(3) } else { plural := "s" if count == 1 { @@ -66,15 +62,15 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, }, } -func runTests(rootPath string, target vespa.Target) (int, []string) { +func runTests(rootPath string, target vespa.Target, dryRun bool) (int, []string) { count := 0 failed := make([]string, 0) if stat, err := os.Stat(rootPath); err != nil { - fatalErr(err, "Failed reading specified test path") + fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") } else if stat.IsDir() { tests, err := ioutil.ReadDir(rootPath) // TODO: Use os.ReadDir when >= 1.16 is required. if err != nil { - fatalErr(err, "Failed reading specified test directory") + fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") } previousFailed := false for _, test := range tests { @@ -84,7 +80,7 @@ func runTests(rootPath string, target vespa.Target) (int, []string) { fmt.Fprintln(stdout, "") previousFailed = false } - failure := runTest(testPath, target) + failure := runTest(testPath, target, dryRun) if failure != "" { failed = append(failed, failure) previousFailed = true @@ -93,64 +89,76 @@ func runTests(rootPath string, target vespa.Target) (int, []string) { } } } else if strings.HasSuffix(stat.Name(), ".json") { - failure := runTest(rootPath, target) + failure := runTest(rootPath, target, dryRun) if failure != "" { failed = append(failed, failure) } count++ } + if count == 0 { + fatalErrHint(fmt.Errorf("Failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing") + } return count, failed } // Runs the test at the given path, and returns the specified test name if the test fails -func runTest(testPath string, target vespa.Target) string { +func runTest(testPath string, target vespa.Target, dryRun bool) string { var test test testBytes, err := ioutil.ReadFile(testPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read test file at %s", testPath)) + fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") } if err = json.Unmarshal(testBytes, &test); err != nil { - fatalErr(err, fmt.Sprintf("Failed to parse test file at %s", testPath)) + fatalErrHint(err, fmt.Sprintf("Failed parsing test at %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") } testName := test.Name if test.Name == "" { testName = filepath.Base(testPath) } - fmt.Fprintf(stdout, "Running %s:", testName) + if !dryRun { + fmt.Fprintf(stdout, "Running %s:", testName) + } defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath)) if err != nil { - fatalErr(err, fmt.Sprintf("Invalid default parameters for %s", testName)) + fmt.Fprintln(stderr) + fatalErrHint(err, fmt.Sprintf("Invalid default parameters for %s", testName), "See https://cloud.vespa.ai/en/reference/testing") } if len(test.Steps) == 0 { - fatalErr(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath)) + fmt.Fprintln(stderr) + fatalErrHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") } for i, step := range test.Steps { stepName := step.Name if stepName == "" { stepName = fmt.Sprintf("step %d", i+1) } - failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target) + failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target, dryRun) if err != nil { - fatalErr(err, fmt.Sprintf("Error in %s", stepName)) + fmt.Fprintln(stderr) + fatalErrHint(err, fmt.Sprintf("Error in %s", stepName), "See https://cloud.vespa.ai/en/reference/testing") } - if failure != "" { - fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure) - return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) - } - if i == 0 { - fmt.Fprintf(stdout, " ") + if !dryRun { + if failure != "" { + fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure) + return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) + } + if i == 0 { + fmt.Fprintf(stdout, " ") + } + fmt.Fprint(stdout, ".") } - fmt.Fprint(stdout, ".") } - fmt.Fprintln(stdout, " OK") + if !dryRun { + fmt.Fprintln(stdout, " OK") + } return "" } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails -func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, string, error) { +func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target, dryRun bool) (string, string, error) { requestBody, err := getBody(step.Request.BodyRaw, testsPath) if err != nil { return "", "", err @@ -171,9 +179,12 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter cluster = defaultCluster } - service, err := target.Service("query", 0, 0, cluster) - if err != nil { - return "", "", err + var service *vespa.Service + if !dryRun { + service, err = target.Service("query", 0, 0, cluster) + if err != nil { + return "", "", err + } } method := step.Request.Method @@ -191,7 +202,11 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter } externalEndpoint := requestUrl.IsAbs() if !externalEndpoint { - requestUrl, err = url.ParseRequestURI(service.BaseURL + requestUri) + baseURL := "http://dummy/" + if service != nil { + baseURL = service.BaseURL + } + requestUrl, err = url.ParseRequestURI(baseURL + requestUri) if err != nil { return "", "", err } @@ -213,6 +228,27 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter } defer request.Body.Close() + statusCode := step.Response.Code + if statusCode == 0 { + statusCode = 200 + } + + responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath) + if err != nil { + return "", "", err + } + var responseBodySpec interface{} + if responseBodySpecBytes != nil { + err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) + if err != nil { + return "", "", fmt.Errorf("invalid response body spec: %w", err) + } + } + + if dryRun { + return "", "", nil + } + var response *http.Response if externalEndpoint { util.ActiveHttpClient.UseCertificate([]tls.Certificate{}) @@ -225,27 +261,14 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter } defer response.Body.Close() - statusCode := step.Response.Code - if statusCode == 0 { - statusCode = 200 - } if statusCode != response.StatusCode { failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode) return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil } - responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath) - if err != nil { - return "", "", err - } - if responseBodySpecBytes == nil { + if responseBodySpec == nil { return "", "", nil } - var responseBodySpec interface{} - err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) - if err != nil { - return "", "", err - } responseBodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { @@ -348,7 +371,7 @@ func getParameters(parametersRaw []byte, testsPath string) (map[string]string, e resolvedParametersPath := path.Join(testsPath, parametersPath) parametersRaw, err = ioutil.ReadFile(resolvedParametersPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read request parameters file at '%s'", resolvedParametersPath)) + return nil, fmt.Errorf("failed to read request parameters at %s: %w", resolvedParametersPath, err) } } var parameters map[string]string @@ -366,7 +389,7 @@ func getBody(bodyRaw []byte, testsPath string) ([]byte, error) { resolvedBodyPath := path.Join(testsPath, bodyPath) bodyRaw, err = ioutil.ReadFile(resolvedBodyPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read body file at '%s'", resolvedBodyPath)) + return nil, fmt.Errorf("failed to read body file at %s: %w", resolvedBodyPath, err) } } return bodyRaw, nil diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index fe04e3a704a..9a566beb10f 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -53,14 +53,13 @@ func TestProductionTest(t *testing.T) { func TestTestWithoutAssertions(t *testing.T) { client := &mockHttpClient{} _, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client) - assert.Equal(t, "a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\n", errBytes) + assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) } func TestSuiteWithoutTests(t *testing.T) { client := &mockHttpClient{} - outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) - assert.Equal(t, "Failed to find any tests at testdata/tests/staging-test\n", outBytes) - assert.Equal(t, "", errBytes) + _, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) + assert.Equal(t, "Error: Failed to find any tests at testdata/tests/staging-test\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) } func TestSingleTest(t *testing.T) { diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index c1cc868e16f..741b45f9e15 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -139,7 +139,7 @@ func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { tempZip.Close() os.Remove(tempZip.Name()) }() - if err := zipDir(ap.Path, tempZip.Name()); err != nil { + if err := zipDir(zipFile, tempZip.Name()); err != nil { return nil, err } zipFile = tempZip.Name() @@ -167,6 +167,10 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application } } if util.PathExists(filepath.Join(zipOrDir, "src", "main", "application")) { + if util.PathExists(filepath.Join(zipOrDir, "src", "test", "application")) { + return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application"), + TestPath: filepath.Join(zipOrDir, "src", "test", "application")}, nil + } return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application")}, nil } if util.PathExists(filepath.Join(zipOrDir, "services.xml")) { @@ -440,7 +444,10 @@ func zipDir(dir string, destination string) error { } defer file.Close() - zippath := strings.TrimPrefix(path, dir) + zippath, err := filepath.Rel(dir, path) + if err != nil { + return err + } zipfile, err := w.Create(zippath) if err != nil { return err |