Merge pull request #20239 from vespa-engine/jonmv/vespa-cli-submit-non-java

Jonmv/vespa cli submit non java

5 files changed, 153 insertions, 65 deletions

diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go
index d1d72362448..89dc4cb6094 100644
--- a/client/go/cmd/prod.go
+++ b/client/go/cmd/prod.go
@@ -116,7 +116,7 @@ For more information about production deployments in Vespa Cloud see:
 https://cloud.vespa.ai/en/getting-to-production
 https://cloud.vespa.ai/en/automated-deployments`,
 	DisableAutoGenTag: true,
-	Example: `$ mvn package
+	Example: `$ mvn package # when adding custom Java components
 $ vespa prod submit`,
 	Run: func(cmd *cobra.Command, args []string) {
 		target := getTarget()
@@ -139,10 +139,13 @@ $ vespa prod submit`,
 			fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init")
 			return
 		}
-		if !pkg.IsJava() {
-			// TODO: Loosen this requirement when we start supporting applications with Java in production
-			fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported")
+		if pkg.TestPath == "" {
+			fatalErrHint(fmt.Errorf("No tests found"),
+				"The application must be a Java maven project, or include basic HTTP tests under src/test/application/",
+				"See https://cloud.vespa.ai/en/reference/getting-to-production")
 			return
+		} else {
+			verifyTests(pkg.TestPath, target)
 		}
 		isCI := os.Getenv("CI") != ""
 		if !isCI {
@@ -347,3 +350,12 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu
 	}
 	return input
 }
+
+func verifyTests(testsParent string, target vespa.Target) {
+	runTests(filepath.Join(testsParent, "tests", "system-test"), target, true)
+	runTests(filepath.Join(testsParent, "tests", "staging-setup"), target, true)
+	runTests(filepath.Join(testsParent, "tests", "staging-test"), target, true)
+	if util.PathExists(filepath.Join(testsParent, "tests", "production-test")) {
+		runTests(filepath.Join(testsParent, "tests", "production-test"), target, true)
+	}
+}
diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go
index 4ce6112122a..a4f3ebd6b56 100644
--- a/client/go/cmd/prod_test.go
+++ b/client/go/cmd/prod_test.go
@@ -16,7 +16,7 @@ import (
 func TestProdInit(t *testing.T) {
 	homeDir := filepath.Join(t.TempDir(), ".vespa")
 	pkgDir := filepath.Join(t.TempDir(), "app")
-	createApplication(t, pkgDir)
+	createApplication(t, pkgDir, false)
 
 	answers := []string{
 		// Regions
@@ -81,7 +81,7 @@ func readFileString(t *testing.T, filename string) string {
 	return string(content)
 }
 
-func createApplication(t *testing.T, pkgDir string) {
+func createApplication(t *testing.T, pkgDir string, java bool) {
 	appDir := filepath.Join(pkgDir, "src", "main", "application")
 	targetDir := filepath.Join(pkgDir, "target")
 	if err := os.MkdirAll(appDir, 0755); err != nil {
@@ -120,7 +120,24 @@ func createApplication(t *testing.T, pkgDir string) {
 	if err := os.MkdirAll(targetDir, 0755); err != nil {
 		t.Fatal(err)
 	}
-	if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil {
+	if java {
+		if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		testsDir := filepath.Join(pkgDir, "src", "test", "application", "tests")
+		testBytes, _ := ioutil.ReadAll(strings.NewReader("{\"steps\":[{}]}"))
+		writeTest(filepath.Join(testsDir, "system-test", "test.json"), testBytes, t)
+		writeTest(filepath.Join(testsDir, "staging-setup", "test.json"), testBytes, t)
+		writeTest(filepath.Join(testsDir, "staging-test", "test.json"), testBytes, t)
+	}
+}
+
+func writeTest(path string, content []byte, t *testing.T) {
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path, content, 0644); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -128,7 +145,37 @@ func createApplication(t *testing.T, pkgDir string) {
 func TestProdSubmit(t *testing.T) {
 	homeDir := filepath.Join(t.TempDir(), ".vespa")
 	pkgDir := filepath.Join(t.TempDir(), "app")
-	createApplication(t, pkgDir)
+	createApplication(t, pkgDir, false)
+
+	httpClient := &mockHttpClient{}
+	httpClient.NextResponse(200, `ok`)
+	execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient)
+	execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient)
+
+	// Zipping requires relative paths, so much let command run from pkgDir, then reset cwd for subsequent tests.
+	if cwd, err := os.Getwd(); err != nil {
+		t.Fatal(err)
+	} else {
+		defer os.Chdir(cwd)
+	}
+	if err := os.Chdir(pkgDir); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Setenv("CI", "true"); err != nil {
+		t.Fatal(err)
+	}
+	out, err := execute(command{homeDir: homeDir, args: []string{"prod", "submit"}}, t, httpClient)
+	assert.Equal(t, "", err)
+	assert.Contains(t, out, "Success: Submitted")
+	assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress")
+}
+
+func TestProdSubmitWithJava(t *testing.T) {
+	homeDir := filepath.Join(t.TempDir(), ".vespa")
+	pkgDir := filepath.Join(t.TempDir(), "app")
+	createApplication(t, pkgDir, true)
 
 	httpClient := &mockHttpClient{}
 	httpClient.NextResponse(200, `ok`)
@@ -137,7 +184,7 @@ func TestProdSubmit(t *testing.T) {
 	execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient)
 	execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient)
 
-	// Copy an application package pre-assambled with mvn package
+	// Copy an application package pre-assembled with mvn package
 	testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target")
 	zipFile := filepath.Join(testAppDir, "application.zip")
 	copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile)
diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go
index d6fdc9d2e41..7c49703595e 100644
--- a/client/go/cmd/test.go
+++ b/client/go/cmd/test.go
@@ -27,16 +27,15 @@ func init() {
 	rootCmd.AddCommand(testCmd)
 }
 
-// TODO: add link to test doc at cloud.vespa.ai
 var testCmd = &cobra.Command{
 	Use:   "test [tests directory or test file]",
 	Short: "Run a test suite, or a single test",
 	Long: `Run a test suite, or a single test
 
-Runs all JSON test files in the specified directory, or the single JSON
-test file specified.
+Runs all JSON test files in the specified directory (the working
+directory by default), or the single JSON test file specified.
 
-If no directory or file is specified, the working directory is used instead.`,
+See https://cloud.vespa.ai/en/reference/testing.html for details.`,
 	Example: `$ vespa test src/test/application/tests/system-test
 $ vespa test src/test/application/tests/system-test/feed-and-query.json`,
 	Args:              cobra.MaximumNArgs(1),
@@ -47,15 +46,12 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`,
 		if len(args) > 0 {
 			testPath = args[0]
 		}
-		if count, failed := runTests(testPath, target); len(failed) != 0 {
+		if count, failed := runTests(testPath, target, false); len(failed) != 0 {
 			fmt.Fprintf(stdout, "\nFailed %d of %d tests:\n", len(failed), count)
 			for _, test := range failed {
 				fmt.Fprintln(stdout, test)
 			}
 			exitFunc(3)
-		} else if count == 0 {
-			fmt.Fprintf(stdout, "Failed to find any tests at %v\n", testPath)
-			exitFunc(3)
 		} else {
 			plural := "s"
 			if count == 1 {
@@ -66,15 +62,15 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`,
 	},
 }
 
-func runTests(rootPath string, target vespa.Target) (int, []string) {
+func runTests(rootPath string, target vespa.Target, dryRun bool) (int, []string) {
 	count := 0
 	failed := make([]string, 0)
 	if stat, err := os.Stat(rootPath); err != nil {
-		fatalErr(err, "Failed reading specified test path")
+		fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
 	} else if stat.IsDir() {
 		tests, err := ioutil.ReadDir(rootPath) // TODO: Use os.ReadDir when >= 1.16 is required.
 		if err != nil {
-			fatalErr(err, "Failed reading specified test directory")
+			fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
 		}
 		previousFailed := false
 		for _, test := range tests {
@@ -84,7 +80,7 @@ func runTests(rootPath string, target vespa.Target) (int, []string) {
 					fmt.Fprintln(stdout, "")
 					previousFailed = false
 				}
-				failure := runTest(testPath, target)
+				failure := runTest(testPath, target, dryRun)
 				if failure != "" {
 					failed = append(failed, failure)
 					previousFailed = true
@@ -93,64 +89,76 @@ func runTests(rootPath string, target vespa.Target) (int, []string) {
 			}
 		}
 	} else if strings.HasSuffix(stat.Name(), ".json") {
-		failure := runTest(rootPath, target)
+		failure := runTest(rootPath, target, dryRun)
 		if failure != "" {
 			failed = append(failed, failure)
 		}
 		count++
 	}
+	if count == 0 {
+		fatalErrHint(fmt.Errorf("Failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing")
+	}
 	return count, failed
 }
 
 // Runs the test at the given path, and returns the specified test name if the test fails
-func runTest(testPath string, target vespa.Target) string {
+func runTest(testPath string, target vespa.Target, dryRun bool) string {
 	var test test
 	testBytes, err := ioutil.ReadFile(testPath)
 	if err != nil {
-		fatalErr(err, fmt.Sprintf("Failed to read test file at %s", testPath))
+		fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing")
 	}
 	if err = json.Unmarshal(testBytes, &test); err != nil {
-		fatalErr(err, fmt.Sprintf("Failed to parse test file at %s", testPath))
+		fatalErrHint(err, fmt.Sprintf("Failed parsing test at %s", testPath), "See https://cloud.vespa.ai/en/reference/testing")
 	}
 
 	testName := test.Name
 	if test.Name == "" {
 		testName = filepath.Base(testPath)
 	}
-	fmt.Fprintf(stdout, "Running %s:", testName)
+	if !dryRun {
+		fmt.Fprintf(stdout, "Running %s:", testName)
+	}
 
 	defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath))
 	if err != nil {
-		fatalErr(err, fmt.Sprintf("Invalid default parameters for %s", testName))
+		fmt.Fprintln(stderr)
+		fatalErrHint(err, fmt.Sprintf("Invalid default parameters for %s", testName), "See https://cloud.vespa.ai/en/reference/testing")
 	}
 
 	if len(test.Steps) == 0 {
-		fatalErr(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath))
+		fmt.Fprintln(stderr)
+		fatalErrHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing")
 	}
 	for i, step := range test.Steps {
 		stepName := step.Name
 		if stepName == "" {
 			stepName = fmt.Sprintf("step %d", i+1)
 		}
-		failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target)
+		failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target, dryRun)
 		if err != nil {
-			fatalErr(err, fmt.Sprintf("Error in %s", stepName))
+			fmt.Fprintln(stderr)
+			fatalErrHint(err, fmt.Sprintf("Error in %s", stepName), "See https://cloud.vespa.ai/en/reference/testing")
 		}
-		if failure != "" {
-			fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure)
-			return fmt.Sprintf("%s: %s: %s", testName, stepName, failure)
-		}
-		if i == 0 {
-			fmt.Fprintf(stdout, " ")
+		if !dryRun {
+			if failure != "" {
+				fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure)
+				return fmt.Sprintf("%s: %s: %s", testName, stepName, failure)
+			}
+			if i == 0 {
+				fmt.Fprintf(stdout, " ")
+			}
+			fmt.Fprint(stdout, ".")
 		}
-		fmt.Fprint(stdout, ".")
 	}
-	fmt.Fprintln(stdout, " OK")
+	if !dryRun {
+		fmt.Fprintln(stdout, " OK")
+	}
 	return ""
 }
 
 // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails
-func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, string, error) {
+func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target, dryRun bool) (string, string, error) {
 	requestBody, err := getBody(step.Request.BodyRaw, testsPath)
 	if err != nil {
 		return "", "", err
@@ -171,9 +179,12 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter
 		cluster = defaultCluster
 	}
 
-	service, err := target.Service("query", 0, 0, cluster)
-	if err != nil {
-		return "", "", err
+	var service *vespa.Service
+	if !dryRun {
+		service, err = target.Service("query", 0, 0, cluster)
+		if err != nil {
+			return "", "", err
+		}
 	}
 
 	method := step.Request.Method
@@ -191,7 +202,11 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter
 	}
 	externalEndpoint := requestUrl.IsAbs()
 	if !externalEndpoint {
-		requestUrl, err = url.ParseRequestURI(service.BaseURL + requestUri)
+		baseURL := "http://dummy/"
+		if service != nil {
+			baseURL = service.BaseURL
+		}
+		requestUrl, err = url.ParseRequestURI(baseURL + requestUri)
 		if err != nil {
 			return "", "", err
 		}
@@ -213,6 +228,27 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter
 	}
 	defer request.Body.Close()
 
+	statusCode := step.Response.Code
+	if statusCode == 0 {
+		statusCode = 200
+	}
+
+	responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath)
+	if err != nil {
+		return "", "", err
+	}
+	var responseBodySpec interface{}
+	if responseBodySpecBytes != nil {
+		err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec)
+		if err != nil {
+			return "", "", fmt.Errorf("invalid response body spec: %w", err)
+		}
+	}
+
+	if dryRun {
+		return "", "", nil
+	}
+
 	var response *http.Response
 	if externalEndpoint {
 		util.ActiveHttpClient.UseCertificate([]tls.Certificate{})
@@ -225,27 +261,14 @@ func verify(step step, testsPath string, defaultCluster string, defaultParameter
 	}
 	defer response.Body.Close()
 
-	statusCode := step.Response.Code
-	if statusCode == 0 {
-		statusCode = 200
-	}
 	if statusCode != response.StatusCode {
 		failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode)
 		return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil
 	}
 
-	responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath)
-	if err != nil {
-		return "", "", err
-	}
-	if responseBodySpecBytes == nil {
+	if responseBodySpec == nil {
 		return "", "", nil
 	}
-	var responseBodySpec interface{}
-	err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec)
-	if err != nil {
-		return "", "", err
-	}
 
 	responseBodyBytes, err := ioutil.ReadAll(response.Body)
 	if err != nil {
@@ -348,7 +371,7 @@ func getParameters(parametersRaw []byte, testsPath string) (map[string]string, e
 			resolvedParametersPath := path.Join(testsPath, parametersPath)
 			parametersRaw, err = ioutil.ReadFile(resolvedParametersPath)
 			if err != nil {
-				fatalErr(err, fmt.Sprintf("Failed to read request parameters file at '%s'", resolvedParametersPath))
+				return nil, fmt.Errorf("failed to read request parameters at %s: %w", resolvedParametersPath, err)
 			}
 		}
 		var parameters map[string]string
@@ -366,7 +389,7 @@ func getBody(bodyRaw []byte, testsPath string) ([]byte, error) {
 		resolvedBodyPath := path.Join(testsPath, bodyPath)
 		bodyRaw, err = ioutil.ReadFile(resolvedBodyPath)
 		if err != nil {
-			fatalErr(err, fmt.Sprintf("Failed to read body file at '%s'", resolvedBodyPath))
+			return nil, fmt.Errorf("failed to read body file at %s: %w", resolvedBodyPath, err)
 		}
 	}
 	return bodyRaw, nil
diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go
index fe04e3a704a..9a566beb10f 100644
--- a/client/go/cmd/test_test.go
+++ b/client/go/cmd/test_test.go
@@ -53,14 +53,13 @@ func TestProductionTest(t *testing.T) {
 func TestTestWithoutAssertions(t *testing.T) {
 	client := &mockHttpClient{}
 	_, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client)
-	assert.Equal(t, "a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\n", errBytes)
+	assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes)
 }
 
 func TestSuiteWithoutTests(t *testing.T) {
 	client := &mockHttpClient{}
-	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client)
-	assert.Equal(t, "Failed to find any tests at testdata/tests/staging-test\n", outBytes)
-	assert.Equal(t, "", errBytes)
+	_, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client)
+	assert.Equal(t, "Error: Failed to find any tests at testdata/tests/staging-test\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes)
 }
 
 func TestSingleTest(t *testing.T) {
diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go
index c1cc868e16f..741b45f9e15 100644
--- a/client/go/vespa/deploy.go
+++ b/client/go/vespa/deploy.go
@@ -139,7 +139,7 @@ func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) {
 			tempZip.Close()
 			os.Remove(tempZip.Name())
 		}()
-		if err := zipDir(ap.Path, tempZip.Name()); err != nil {
+		if err := zipDir(zipFile, tempZip.Name()); err != nil {
 			return nil, err
 		}
 		zipFile = tempZip.Name()
@@ -167,6 +167,10 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application
 		}
 	}
 	if util.PathExists(filepath.Join(zipOrDir, "src", "main", "application")) {
+		if util.PathExists(filepath.Join(zipOrDir, "src", "test", "application")) {
+			return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application"),
+				TestPath: filepath.Join(zipOrDir, "src", "test", "application")}, nil
+		}
 		return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application")}, nil
 	}
 	if util.PathExists(filepath.Join(zipOrDir, "services.xml")) {
@@ -440,7 +444,10 @@ func zipDir(dir string, destination string) error {
 		}
 		defer file.Close()
 
-		zippath := strings.TrimPrefix(path, dir)
+		zippath, err := filepath.Rel(dir, path)
+		if err != nil {
+			return err
+		}
 		zipfile, err := w.Create(zippath)
 		if err != nil {
 			return err