diff options
33 files changed, 680 insertions, 368 deletions
diff --git a/client/go/.gitignore b/client/go/.gitignore index eb679add05e..8933bc220cb 100644 --- a/client/go/.gitignore +++ b/client/go/.gitignore @@ -4,3 +4,5 @@ share/ !Makefile !build/ !target/ +mytestapp/ +mytestapp-cache/ diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index 5cc1dab8a35..ae3f5346f4c 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -79,11 +79,19 @@ func doApiKey(_ *cobra.Command, _ []string) error { if err != nil { return err } + targetType, err := getTargetType() + if err != nil { + return err + } + system, err := getSystem(targetType) + if err != nil { + return err + } apiKeyFile := cfg.APIKeyPath(app.Tenant) if util.PathExists(apiKeyFile) && !overwriteKey { err := fmt.Errorf("refusing to overwrite %s", apiKeyFile) printErrHint(err, "Use -f to overwrite it") - printPublicKey(apiKeyFile, app.Tenant) + printPublicKey(system, apiKeyFile, app.Tenant) return ErrCLI{error: err, quiet: true} } apiKey, err := vespa.CreateAPIKey() @@ -92,13 +100,13 @@ func doApiKey(_ *cobra.Command, _ []string) error { } if err := ioutil.WriteFile(apiKeyFile, apiKey, 0600); err == nil { printSuccess("API private key written to ", apiKeyFile) - return printPublicKey(apiKeyFile, app.Tenant) + return printPublicKey(system, apiKeyFile, app.Tenant) } else { return fmt.Errorf("failed to write: %s: %w", apiKeyFile, err) } } -func printPublicKey(apiKeyFile, tenant string) error { +func printPublicKey(system vespa.System, apiKeyFile, tenant string) error { pemKeyData, err := ioutil.ReadFile(apiKeyFile) if err != nil { return fmt.Errorf("failed to read: %s: %w", apiKeyFile, err) @@ -118,7 +126,7 @@ func printPublicKey(apiKeyFile, tenant string) error { log.Printf("\nThis is your public key:\n%s", color.Green(pemPublicKey)) log.Printf("Its fingerprint is:\n%s\n", color.Cyan(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") - log.Printf(color.Cyan("%s/tenant/%s/keys").String(), getConsoleURL(), tenant) + log.Printf(color.Cyan("%s/tenant/%s/keys").String(), system.ConsoleURL, tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") return nil } diff --git a/client/go/cmd/api_key_test.go b/client/go/cmd/api_key_test.go index 935b8676c09..ba697b69d9f 100644 --- a/client/go/cmd/api_key_test.go +++ b/client/go/cmd/api_key_test.go @@ -23,6 +23,8 @@ func testAPIKey(t *testing.T, subcommand []string) { homeDir := filepath.Join(t.TempDir(), ".vespa") keyFile := filepath.Join(homeDir, "t1.api-key.pem") + execute(command{args: []string{"config", "set", "target", "cloud"}, homeDir: homeDir}, t, nil) + args := append(subcommand, "-a", "t1.a1.i1") out, _ := execute(command{args: args, homeDir: homeDir}, t, nil) assert.Contains(t, out, "Success: API private key written to "+keyFile+"\n") diff --git a/client/go/cmd/clone_list_test.go b/client/go/cmd/clone_list_test.go index 1138e5de064..2e4fc4004bd 100644 --- a/client/go/cmd/clone_list_test.go +++ b/client/go/cmd/clone_list_test.go @@ -7,11 +7,12 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" ) func TestListSampleApps(t *testing.T) { - c := &mockHttpClient{} + c := &mock.HTTPClient{} c.NextResponse(200, readTestData(t, "sample-apps-contents.json")) c.NextResponse(200, readTestData(t, "sample-apps-news.json")) c.NextResponse(200, readTestData(t, "sample-apps-operations.json")) diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go index 18354ece0e1..332758a127a 100644 --- a/client/go/cmd/clone_test.go +++ b/client/go/cmd/clone_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" ) @@ -21,7 +22,7 @@ func TestClone(t *testing.T) { func assertCreated(sampleAppName string, app string, t *testing.T) { appCached := app + "-cache" - httpClient := &mockHttpClient{} + httpClient := &mock.HTTPClient{} testdata, err := ioutil.ReadFile(filepath.Join("testdata", "sample-apps-master.zip")) require.Nil(t, err) httpClient.NextResponseBytes(200, testdata) diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester_test.go index 2391cb82d15..d71cdde0e8f 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester_test.go @@ -6,19 +6,15 @@ package cmd import ( "bytes" - "crypto/tls" "io" - "io/ioutil" - "net/http" "os" "path/filepath" - "strconv" "testing" - "time" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" ) @@ -66,7 +62,7 @@ func resetEnv(env map[string]string, original map[string]string) { } } -func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) { +func execute(cmd command, t *testing.T, client *mock.HTTPClient) (string, string) { if client != nil { util.ActiveHttpClient = client } @@ -122,52 +118,7 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) return capturedOut.String(), capturedErr.String() } -func executeCommand(t *testing.T, client *mockHttpClient, args []string, moreArgs []string) string { +func executeCommand(t *testing.T, client *mock.HTTPClient, args []string, moreArgs []string) string { out, _ := execute(command{args: args, moreArgs: moreArgs}, t, client) return out } - -type mockHttpClient struct { - // The responses to return for future requests. Once a response is consumed, it's removed from this array - nextResponses []mockResponse - - // A recording of the last HTTP request made through this - lastRequest *http.Request - - // All requests made through this - requests []*http.Request -} - -type mockResponse struct { - status int - body []byte -} - -func (c *mockHttpClient) NextStatus(status int) { c.NextResponseBytes(status, nil) } - -func (c *mockHttpClient) NextResponse(status int, body string) { - c.NextResponseBytes(status, []byte(body)) -} - -func (c *mockHttpClient) NextResponseBytes(status int, body []byte) { - c.nextResponses = append(c.nextResponses, mockResponse{status: status, body: body}) -} - -func (c *mockHttpClient) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { - response := mockResponse{status: 200} - if len(c.nextResponses) > 0 { - response = c.nextResponses[0] - c.nextResponses = c.nextResponses[1:] - } - c.lastRequest = request - c.requests = append(c.requests, request) - return &http.Response{ - Status: "Status " + strconv.Itoa(response.status), - StatusCode: response.status, - Body: ioutil.NopCloser(bytes.NewBuffer(response.body)), - Header: make(http.Header), - }, - nil -} - -func (c *mockHttpClient) UseCertificate(certificates []tls.Certificate) {} diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 9d42f0683fd..0997be2c899 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -197,7 +197,7 @@ func (c *Config) ReadAPIKey(tenantName string) ([]byte, error) { } // UseAPIKey checks if api key should be used be checking if api-key or api-key-file has been set. -func (c *Config) UseAPIKey(tenantName string) bool { +func (c *Config) UseAPIKey(system vespa.System, tenantName string) bool { if _, err := c.Get(apiKeyFlag); err == nil { return true } @@ -207,15 +207,11 @@ func (c *Config) UseAPIKey(tenantName string) bool { // If no Auth0 token is created, fall back to tenant api key, but warn that this functionality is deprecated // TODO: Remove this when users have had time to migrate over to Auth0 device flow authentication - a, err := auth0.GetAuth0(c.AuthConfigPath(), getSystemName(), getApiURL()) + a, err := auth0.GetAuth0(c.AuthConfigPath(), system.Name, system.URL) if err != nil || !a.HasSystem() { fmt.Fprintln(stderr, "Defaulting to tenant API key is deprecated. Use Auth0 device flow: 'vespa auth login' instead") - if !util.PathExists(c.APIKeyPath(tenantName)) { - return false - } - return true + return util.PathExists(c.APIKeyPath(tenantName)) } - return false } @@ -283,7 +279,7 @@ func (c *Config) Set(option, value string) error { switch option { case targetFlag: switch value { - case "local", "cloud": + case vespa.TargetLocal, vespa.TargetCloud, vespa.TargetHosted: viper.Set(option, value) return nil } diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index 16378b5f8ba..2f0ccbb29e1 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/vespa" ) func TestConfig(t *testing.T) { @@ -16,6 +17,8 @@ func TestConfig(t *testing.T) { assertConfigCommandErr(t, "Error: invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") assertConfigCommand(t, "foo = <unset>\n", homeDir, "config", "get", "foo") assertConfigCommand(t, "target = local\n", homeDir, "config", "get", "target") + assertConfigCommand(t, "", homeDir, "config", "set", "target", "hosted") + assertConfigCommand(t, "target = hosted\n", homeDir, "config", "get", "target") assertConfigCommand(t, "", homeDir, "config", "set", "target", "cloud") assertConfigCommand(t, "target = cloud\n", homeDir, "config", "get", "target") assertConfigCommand(t, "", homeDir, "config", "set", "target", "http://127.0.0.1:8080") @@ -66,15 +69,15 @@ func TestUseAPIKey(t *testing.T) { homeDir := t.TempDir() c := Config{Home: homeDir} - assert.False(t, c.UseAPIKey("t1")) + assert.False(t, c.UseAPIKey(vespa.PublicSystem, "t1")) c.Set(apiKeyFileFlag, "/tmp/foo") - assert.True(t, c.UseAPIKey("t1")) + assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t1")) c.Set(apiKeyFileFlag, "") withEnv("VESPA_CLI_API_KEY", "...", func() { require.Nil(t, c.load()) - assert.True(t, c.UseAPIKey("t1")) + assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t1")) }) // Test deprecated functionality @@ -97,8 +100,8 @@ func TestUseAPIKey(t *testing.T) { withEnv("VESPA_CLI_CLOUD_SYSTEM", "public", func() { _, err := os.Create(filepath.Join(homeDir, "t2.api-key.pem")) require.Nil(t, err) - assert.True(t, c.UseAPIKey("t2")) + assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t2")) require.Nil(t, ioutil.WriteFile(filepath.Join(homeDir, "auth.json"), []byte(authContent), 0600)) - assert.False(t, c.UseAPIKey("t2")) + assert.False(t, c.UseAPIKey(vespa.PublicSystem, "t2")) }) } diff --git a/client/go/cmd/curl.go b/client/go/cmd/curl.go index b66780780ed..1ede2cccae3 100644 --- a/client/go/cmd/curl.go +++ b/client/go/cmd/curl.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/auth0" "github.com/vespa-engine/vespa/client/go/curl" + "github.com/vespa-engine/vespa/client/go/vespa" ) var curlDryRun bool @@ -61,8 +62,8 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain if err != nil { return err } - if t.Type() == "cloud" { - if err := addCloudAuth0Authentication(cfg, c); err != nil { + if t.Type() == vespa.TargetCloud { + if err := addCloudAuth0Authentication(t.Deployment().System, cfg, c); err != nil { return err } } @@ -92,17 +93,17 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain }, } -func addCloudAuth0Authentication(cfg *Config, c *curl.Command) error { - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL()) +func addCloudAuth0Authentication(system vespa.System, cfg *Config, c *curl.Command) error { + a, err := auth0.GetAuth0(cfg.AuthConfigPath(), system.Name, system.URL) if err != nil { return err } - system, err := a.PrepareSystem(auth0.ContextWithCancel()) + authSystem, err := a.PrepareSystem(auth0.ContextWithCancel()) if err != nil { return err } - c.Header("Authorization", "Bearer "+system.AccessToken) + c.Header("Authorization", "Bearer "+authSystem.AccessToken) return nil } diff --git a/client/go/cmd/curl_test.go b/client/go/cmd/curl_test.go index 5709096fe37..253943f2b04 100644 --- a/client/go/cmd/curl_test.go +++ b/client/go/cmd/curl_test.go @@ -7,11 +7,12 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" ) func TestCurl(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") - httpClient := &mockHttpClient{} + httpClient := &mock.HTTPClient{} out, _ := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "-a", "t1.a1.i1", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) expected := fmt.Sprintf("curl --key %s --cert %s -v --data-urlencode 'arg=with space' http://127.0.0.1:8080/search\n", diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 14b6e969df7..13f37fa3901 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -26,7 +26,7 @@ func init() { rootCmd.AddCommand(deployCmd) rootCmd.AddCommand(prepareCmd) rootCmd.AddCommand(activateCmd) - deployCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "dev.aws-us-east-1c", "The zone to use for deployment") + deployCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "", "The zone to use for deployment. This defaults to a dev zone") deployCmd.PersistentFlags().StringVarP(&logLevelArg, logLevelFlag, "l", "error", `Log level for Vespa logs. Must be "error", "warning", "info" or "debug"`) } @@ -64,7 +64,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, if err != nil { return err } - opts, err := getDeploymentOpts(cfg, pkg, target) + opts, err := getDeploymentOptions(cfg, pkg, target) if err != nil { return err } @@ -73,7 +73,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return err } - fmt.Print("\n") + log.Println() if opts.IsCloud() { printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) } else { @@ -82,9 +82,9 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, if opts.IsCloud() { log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", - getConsoleURL(), - opts.Deployment.Application.Tenant, opts.Deployment.Application.Application, opts.Deployment.Application.Instance, - opts.Deployment.Zone.Environment, opts.Deployment.Zone.Region, + opts.Target.Deployment().System.ConsoleURL, + opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application, opts.Target.Deployment().Application.Instance, + opts.Target.Deployment().Zone.Environment, opts.Target.Deployment().Zone.Region, sessionOrRunID))) } return waitForQueryService(sessionOrRunID) @@ -110,7 +110,7 @@ var prepareCmd = &cobra.Command{ if err != nil { return err } - sessionID, err := vespa.Prepare(vespa.DeploymentOpts{ + sessionID, err := vespa.Prepare(vespa.DeploymentOptions{ ApplicationPackage: pkg, Target: target, }) @@ -148,7 +148,7 @@ var activateCmd = &cobra.Command{ if err != nil { return err } - err = vespa.Activate(sessionID, vespa.DeploymentOpts{ + err = vespa.Activate(sessionID, vespa.DeploymentOptions{ ApplicationPackage: pkg, Target: target, }) diff --git a/client/go/cmd/deploy_test.go b/client/go/cmd/deploy_test.go index a37a433397f..f5af3751eb8 100644 --- a/client/go/cmd/deploy_test.go +++ b/client/go/cmd/deploy_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -32,9 +33,9 @@ func TestDeployZipWithURLTargetArgument(t *testing.T) { applicationPackage := "testdata/applications/withTarget/target/application.zip" arguments := []string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "http://target:19071"} - client := &mockHttpClient{} + client := &mock.HTTPClient{} assert.Equal(t, - "Success: Deployed "+applicationPackage+"\n", + "\nSuccess: Deployed "+applicationPackage+"\n", executeCommand(t, client, arguments, []string{})) assertDeployRequestMade("http://target:19071", client, t) } @@ -60,7 +61,7 @@ func TestDeployApplicationDirectoryWithPomAndTarget(t *testing.T) { } func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withEmptyTarget"}}, t, client) assert.Equal(t, "Error: pom.xml exists but no target/application.zip. Run mvn package first\n", @@ -104,15 +105,15 @@ func TestDeployError(t *testing.T) { } func assertDeploy(applicationPackage string, arguments []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} assert.Equal(t, - "Success: Deployed "+applicationPackage+"\n", + "\nSuccess: Deployed "+applicationPackage+"\n", executeCommand(t, client, arguments, []string{})) assertDeployRequestMade("http://127.0.0.1:19071", client, t) } func assertPrepare(applicationPackage string, arguments []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(200, `{"session-id":"42"}`) assert.Equal(t, "Success: Prepared "+applicationPackage+" with session 42\n", @@ -120,12 +121,12 @@ func assertPrepare(applicationPackage string, arguments []string, t *testing.T) assertPackageUpload(0, "http://127.0.0.1:19071/application/v2/tenant/default/session", client, t) sessionURL := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/prepared" - assert.Equal(t, sessionURL, client.requests[1].URL.String()) - assert.Equal(t, "PUT", client.requests[1].Method) + assert.Equal(t, sessionURL, client.Requests[1].URL.String()) + assert.Equal(t, "PUT", client.Requests[1].Method) } func assertActivate(applicationPackage string, arguments []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} homeDir := t.TempDir() cfg := Config{Home: filepath.Join(homeDir, ".vespa"), createDirs: true} if err := cfg.WriteSessionID(vespa.DefaultApplication, 42); err != nil { @@ -136,14 +137,14 @@ func assertActivate(applicationPackage string, arguments []string, t *testing.T) "Success: Activated "+applicationPackage+" with session 42\n", out) url := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/active" - assert.Equal(t, url, client.lastRequest.URL.String()) - assert.Equal(t, "PUT", client.lastRequest.Method) + assert.Equal(t, url, client.LastRequest.URL.String()) + assert.Equal(t, "PUT", client.LastRequest.Method) } -func assertPackageUpload(requestNumber int, url string, client *mockHttpClient, t *testing.T) { - req := client.lastRequest +func assertPackageUpload(requestNumber int, url string, client *mock.HTTPClient, t *testing.T) { + req := client.LastRequest if requestNumber >= 0 { - req = client.requests[requestNumber] + req = client.Requests[requestNumber] } assert.Equal(t, url, req.URL.String()) assert.Equal(t, "application/zip", req.Header.Get("Content-Type")) @@ -155,12 +156,12 @@ func assertPackageUpload(requestNumber int, url string, client *mockHttpClient, assert.Equal(t, "PK\x03\x04\x14\x00\b", string(buf)) } -func assertDeployRequestMade(target string, client *mockHttpClient, t *testing.T) { +func assertDeployRequestMade(target string, client *mock.HTTPClient, t *testing.T) { assertPackageUpload(-1, target+"/application/v2/tenant/default/prepareandactivate", client, t) } func assertApplicationPackageError(t *testing.T, cmd string, status int, expectedMessage string, returnBody string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(status, returnBody) _, outErr := execute(command{args: []string{cmd, "testdata/applications/withTarget/target/application.zip"}}, t, client) assert.Equal(t, @@ -169,10 +170,10 @@ func assertApplicationPackageError(t *testing.T, cmd string, status int, expecte } func assertDeployServerError(t *testing.T, status int, errorMessage string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withTarget/target/application.zip"}}, t, client) assert.Equal(t, - "Error: error from deploy service at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", + "Error: error from deploy api at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", outErr) } diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index 2b596e9893b..1d650f77d08 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" ) @@ -66,7 +67,7 @@ func TestDocumentRemoveWithoutIdArg(t *testing.T) { func TestDocumentSendMissingId(t *testing.T) { arguments := []string{"document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json"} - client := &mockHttpClient{} + client := &mock.HTTPClient{} _, outErr := execute(command{args: arguments}, t, client) assert.Equal(t, "Error: No document id given neither as argument or as a 'put' key in the json file\n", @@ -75,7 +76,7 @@ func TestDocumentSendMissingId(t *testing.T) { func TestDocumentSendWithDisagreeingOperations(t *testing.T) { arguments := []string{"document", "update", "testdata/A-Head-Full-of-Dreams-Put.json"} - client := &mockHttpClient{} + client := &mock.HTTPClient{} _, outErr := execute(command{args: arguments}, t, client) assert.Equal(t, "Error: Wanted document operation is update but the JSON file specifies put\n", @@ -96,7 +97,7 @@ func TestDocumentGet(t *testing.T) { } func assertDocumentSend(arguments []string, expectedOperation string, expectedMethod string, expectedDocumentId string, expectedPayloadFile string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} documentURL, err := documentServiceURL(client) if err != nil { t.Fatal(err) @@ -116,16 +117,16 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe assert.Equal(t, expectedCurl, errOut) } assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", out) - assert.Equal(t, expectedURL, client.lastRequest.URL.String()) - assert.Equal(t, "application/json", client.lastRequest.Header.Get("Content-Type")) - assert.Equal(t, expectedMethod, client.lastRequest.Method) + assert.Equal(t, expectedURL, client.LastRequest.URL.String()) + assert.Equal(t, "application/json", client.LastRequest.Header.Get("Content-Type")) + assert.Equal(t, expectedMethod, client.LastRequest.Method) expectedPayload, _ := ioutil.ReadFile(expectedPayloadFile) - assert.Equal(t, string(expectedPayload), util.ReaderToString(client.lastRequest.Body)) + assert.Equal(t, string(expectedPayload), util.ReaderToString(client.LastRequest.Body)) } func assertDocumentGet(arguments []string, documentId string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} documentURL, err := documentServiceURL(client) if err != nil { t.Fatal(err) @@ -140,12 +141,12 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { `, executeCommand(t, client, arguments, []string{})) expectedPath, _ := vespa.IdToURLPath(documentId) - assert.Equal(t, documentURL+"/document/v1/"+expectedPath, client.lastRequest.URL.String()) - assert.Equal(t, "GET", client.lastRequest.Method) + assert.Equal(t, documentURL+"/document/v1/"+expectedPath, client.LastRequest.URL.String()) + assert.Equal(t, "GET", client.LastRequest.Method) } func assertDocumentError(t *testing.T, status int, errorMessage string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"document", "put", "id:mynamespace:music::a-head-full-of-dreams", @@ -156,7 +157,7 @@ func assertDocumentError(t *testing.T, status int, errorMessage string) { } func assertDocumentServerError(t *testing.T, status int, errorMessage string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"document", "put", "id:mynamespace:music::a-head-full-of-dreams", @@ -166,7 +167,7 @@ func assertDocumentServerError(t *testing.T, status int, errorMessage string) { outErr) } -func documentServiceURL(client *mockHttpClient) (string, error) { +func documentServiceURL(client *mock.HTTPClient) (string, error) { service, err := getService("document", 0, "") if err != nil { return "", err diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 547126a8156..ab47a0e6d88 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -5,6 +5,8 @@ package cmd import ( + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "log" @@ -29,6 +31,40 @@ func printSuccess(msg ...interface{}) { log.Print(color.Green("Success: "), fmt.Sprint(msg...)) } +func athenzPath(filename string) (string, error) { + userHome, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(userHome, ".athenz", filename), nil +} + +func athenzKeyPair() (tls.Certificate, error) { + certFile, err := athenzPath("cert") + if err != nil { + return tls.Certificate{}, err + } + keyFile, err := athenzPath("key") + if err != nil { + return tls.Certificate{}, err + } + kp, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return tls.Certificate{}, err + } + cert, err := x509.ParseCertificate(kp.Certificate[0]) + if err != nil { + return tls.Certificate{}, err + } + now := time.Now() + expiredAt := cert.NotAfter + if expiredAt.Before(now) { + delta := now.Sub(expiredAt).Truncate(time.Second) + return tls.Certificate{}, errHint(fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta), "Try renewing certificate with 'athenz-user-cert'") + } + return kp, nil +} + func vespaCliHome() (string, error) { home := os.Getenv("VESPA_CLI_HOME") if home == "" { @@ -59,16 +95,20 @@ func vespaCliCacheDir() (string, error) { return cacheDir, nil } -func deploymentFromArgs() (vespa.Deployment, error) { - zone, err := vespa.ZoneFromString(zoneArg) - if err != nil { - return vespa.Deployment{}, err +func deploymentFromArgs(system vespa.System) (vespa.Deployment, error) { + zone := system.DefaultZone + var err error + if zoneArg != "" { + zone, err = vespa.ZoneFromString(zoneArg) + if err != nil { + return vespa.Deployment{}, err + } } app, err := getApplication() if err != nil { return vespa.Deployment{}, err } - return vespa.Deployment{Application: app, Zone: zone}, nil + return vespa.Deployment{System: system, Application: app, Zone: zone}, nil } func applicationSource(args []string) string { @@ -124,28 +164,18 @@ func getService(service string, sessionOrRunID int64, cluster string) (*vespa.Se func getEndpointsOverride() string { return os.Getenv("VESPA_CLI_ENDPOINTS") } -func getSystem() string { return os.Getenv("VESPA_CLI_CLOUD_SYSTEM") } - -func getSystemName() string { - if getSystem() == "publiccd" { - return "publiccd" +func getSystem(targetType string) (vespa.System, error) { + name := os.Getenv("VESPA_CLI_CLOUD_SYSTEM") + if name != "" { + return vespa.GetSystem(name) } - return "public" -} - -func getConsoleURL() string { - if getSystem() == "publiccd" { - return "https://console-cd.vespa.oath.cloud" - } - return "https://console.vespa.oath.cloud" - -} - -func getApiURL() string { - if getSystem() == "publiccd" { - return "https://api.vespa-external-cd.aws.oath.cloud:4443" + switch targetType { + case vespa.TargetHosted: + return vespa.MainSystem, nil + case vespa.TargetCloud: + return vespa.PublicSystem, nil } - return "https://api.vespa-external.aws.oath.cloud:4443" + return vespa.System{}, fmt.Errorf("no default system found for %s target", targetType) } func getTarget() (vespa.Target, error) { @@ -172,53 +202,80 @@ func createTarget() (vespa.Target, error) { return vespa.CustomTarget(targetType), nil } switch targetType { - case "local": + case vespa.TargetLocal: return vespa.LocalTarget(), nil - case "cloud": - cfg, err := LoadConfig() - if err != nil { - return nil, err - } - deployment, err := deploymentFromArgs() - if err != nil { - return nil, err - } - endpoints, err := getEndpointsFromEnv() - if err != nil { - return nil, err - } + case vespa.TargetCloud, vespa.TargetHosted: + return createCloudTarget(targetType) + } + return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") +} - var apiKey []byte = nil - if cfg.UseAPIKey(deployment.Application.Tenant) { +func createCloudTarget(targetType string) (vespa.Target, error) { + cfg, err := LoadConfig() + if err != nil { + return nil, err + } + system, err := getSystem(targetType) + if err != nil { + return nil, err + } + deployment, err := deploymentFromArgs(system) + if err != nil { + return nil, err + } + endpoints, err := getEndpointsFromEnv() + if err != nil { + return nil, err + } + var ( + apiKey []byte + authConfigPath string + apiTLSOptions vespa.TLSOptions + deploymentTLSOptions vespa.TLSOptions + ) + if targetType == vespa.TargetCloud { + if cfg.UseAPIKey(system, deployment.Application.Tenant) { apiKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) if err != nil { return nil, err } } + authConfigPath = cfg.AuthConfigPath() kp, err := cfg.X509KeyPair(deployment.Application) if err != nil { return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") } - - return vespa.CloudTarget( - getApiURL(), - deployment, - apiKey, - vespa.TLSOptions{ - KeyPair: kp.KeyPair, - CertificateFile: kp.CertificateFile, - PrivateKeyFile: kp.PrivateKeyFile, - }, - vespa.LogOptions{ - Writer: stdout, - Level: vespa.LogLevel(logLevelArg), - }, - cfg.AuthConfigPath(), - getSystemName(), - endpoints, - ), nil - } - return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL") + deploymentTLSOptions = vespa.TLSOptions{ + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, + } + } else if targetType == vespa.TargetHosted { + kp, err := athenzKeyPair() + if err != nil { + return nil, err + } + apiTLSOptions = vespa.TLSOptions{KeyPair: kp} + deploymentTLSOptions = apiTLSOptions + } else { + return nil, fmt.Errorf("invalid cloud target: %s", targetType) + } + apiOptions := vespa.APIOptions{ + System: system, + TLSOptions: apiTLSOptions, + APIKey: apiKey, + AuthConfigPath: authConfigPath, + } + deploymentOptions := vespa.CloudDeploymentOptions{ + Deployment: deployment, + TLSOptions: deploymentTLSOptions, + ClusterURLs: endpoints, + } + logOptions := vespa.LogOptions{ + Writer: stdout, + Level: vespa.LogLevel(logLevelArg), + } + return vespa.CloudTarget(apiOptions, deploymentOptions, logOptions) } func waitForService(service string, sessionOrRunID int64) error { @@ -242,25 +299,15 @@ func waitForService(service string, sessionOrRunID int64) error { return nil } -func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) (vespa.DeploymentOpts, error) { - opts := vespa.DeploymentOpts{ApplicationPackage: pkg, Target: target} +func getDeploymentOptions(cfg *Config, pkg vespa.ApplicationPackage, target vespa.Target) (vespa.DeploymentOptions, error) { + opts := vespa.DeploymentOptions{ApplicationPackage: pkg, Target: target} if opts.IsCloud() { - deployment, err := deploymentFromArgs() - if err != nil { - return vespa.DeploymentOpts{}, err - } - if !opts.ApplicationPackage.HasCertificate() { + if target.Type() == vespa.TargetCloud && !opts.ApplicationPackage.HasCertificate() { hint := "Try 'vespa auth cert'" - return vespa.DeploymentOpts{}, errHint(fmt.Errorf("missing certificate in application package"), "Applications in Vespa Cloud require a certificate", hint) - } - if cfg.UseAPIKey(deployment.Application.Tenant) { - opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant) - if err != nil { - return vespa.DeploymentOpts{}, err - } + return vespa.DeploymentOptions{}, errHint(fmt.Errorf("missing certificate in application package"), "Applications in Vespa Cloud require a certificate", hint) } - opts.Deployment = deployment } + opts.Timeout = time.Duration(waitSecsArg) * time.Second return opts, nil } diff --git a/client/go/cmd/log_test.go b/client/go/cmd/log_test.go index 1208be8f80c..3f6714b0d3c 100644 --- a/client/go/cmd/log_test.go +++ b/client/go/cmd/log_test.go @@ -7,20 +7,23 @@ import ( "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/build" + "github.com/vespa-engine/vespa/client/go/mock" ) func TestLog(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, false) - httpClient := &mockHttpClient{} + httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"config", "set", "api-key-file", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) - out, _ := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z"}}, t, httpClient) + out, outErr := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z"}}, t, httpClient) + assert.Equal(t, "", outErr) expected := "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication Switching to the latest deployed set of configurations and components. Application config generation: 52532\n" assert.Equal(t, expected, out) @@ -34,13 +37,14 @@ func TestLogOldClient(t *testing.T) { build.Version = "7.0.0" homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := mockApplicationPackage(t, false) - httpClient := &mockHttpClient{} + httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `{"minVersion": "8.0.0"}`) execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"config", "set", "api-key-file", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) out, errOut := execute(command{homeDir: homeDir, args: []string{"log"}}, t, httpClient) assert.Equal(t, "", out) expected := "Error: client version 7.0.0 is less than the minimum supported version: 8.0.0\nHint: This is not a fatal error, but this version may not work as expected\nHint: Try 'vespa version' to check for a new version\n" diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go index 8787f1f80f5..2ac480d05f5 100644 --- a/client/go/cmd/login.go +++ b/client/go/cmd/login.go @@ -18,7 +18,15 @@ var loginCmd = &cobra.Command{ if err != nil { return err } - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL()) + targetType, err := getTargetType() + if err != nil { + return err + } + system, err := getSystem(targetType) + if err != nil { + return err + } + a, err := auth0.GetAuth0(cfg.AuthConfigPath(), system.Name, system.URL) if err != nil { return err } diff --git a/client/go/cmd/logout.go b/client/go/cmd/logout.go index ddc1d36d5e1..b1f2477aba4 100644 --- a/client/go/cmd/logout.go +++ b/client/go/cmd/logout.go @@ -17,7 +17,15 @@ var logoutCmd = &cobra.Command{ if err != nil { return err } - a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL()) + targetType, err := getTargetType() + if err != nil { + return err + } + system, err := getSystem(targetType) + if err != nil { + return err + } + a, err := auth0.GetAuth0(cfg.AuthConfigPath(), system.Name, system.URL) if err != nil { return err } diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index 8c40eb969bf..10fc9f92368 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -73,6 +73,10 @@ https://cloud.vespa.ai/en/reference/deployment`, if err != nil { return fmt.Errorf("a services.xml declaring your cluster(s) must exist: %w", err) } + target, err := getTarget() + if err != nil { + return err + } fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"), "!\nBefore modification a backup of the original file will be created.\n\n") @@ -80,7 +84,7 @@ https://cloud.vespa.ai/en/reference/deployment`, fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.Green("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) r := bufio.NewReader(stdin) - deploymentXML, err = updateRegions(r, deploymentXML) + deploymentXML, err = updateRegions(r, deploymentXML, target.Deployment().System) if err != nil { return err } @@ -127,8 +131,9 @@ $ vespa prod submit`, if err != nil { return err } - if target.Type() != "cloud" { - return fmt.Errorf("%s target cannot deploy to Vespa Cloud", target.Type()) + if target.Type() != vespa.TargetCloud { + // TODO: Add support for hosted + return fmt.Errorf("prod submit does not support %s target", target.Type()) } appSource := applicationSource(args) pkg, err := vespa.FindApplicationPackage(appSource, true) @@ -156,7 +161,7 @@ $ vespa prod submit`, fmt.Fprintln(stderr, color.Yellow("Warning:"), "We recommend doing this only from a CD job") printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production") } - opts, err := getDeploymentOpts(cfg, pkg, target) + opts, err := getDeploymentOptions(cfg, pkg, target) if err != nil { return err } @@ -165,7 +170,7 @@ $ vespa prod submit`, } else { printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment") log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", - getConsoleURL(), opts.Deployment.Application.Tenant, opts.Deployment.Application.Application))) + opts.Target.Deployment().System.ConsoleURL, opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application))) } return nil }, @@ -202,8 +207,8 @@ func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) er return ioutil.WriteFile(dst, []byte(contents), 0644) } -func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) (xml.Deployment, error) { - regions, err := promptRegions(r, deploymentXML) +func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (xml.Deployment, error) { + regions, err := promptRegions(r, deploymentXML, system) if err != nil { return xml.Deployment{}, err } @@ -222,7 +227,7 @@ func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment) (xml.Deploymen return deploymentXML, nil } -func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) (string, error) { +func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (string, error) { fmt.Fprintln(stdout, color.Cyan("> Deployment regions")) fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones")) fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a")) @@ -238,7 +243,7 @@ func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment) (string, error validator := func(input string) error { regions := strings.Split(input, ",") for _, r := range regions { - if !xml.IsProdRegion(r, getSystem()) { + if !xml.IsProdRegion(r, system) { return fmt.Errorf("invalid region %s", r) } } diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 8fa9ef401b5..90b67af8669 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" ) @@ -147,12 +148,12 @@ func TestProdSubmit(t *testing.T) { pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, false) - httpClient := &mockHttpClient{} + httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `ok`) execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) // Zipping requires relative paths, so much let command run from pkgDir, then reset cwd for subsequent tests. if cwd, err := os.Getwd(); err != nil { @@ -166,8 +167,8 @@ func TestProdSubmit(t *testing.T) { if err := os.Setenv("CI", "true"); err != nil { t.Fatal(err) } - out, err := execute(command{homeDir: homeDir, args: []string{"prod", "submit", "-k", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) - assert.Equal(t, "", err) + out, outErr := execute(command{homeDir: homeDir, args: []string{"prod", "submit", "-k", filepath.Join(homeDir, "t1.api-key.pem")}}, t, httpClient) + assert.Equal(t, "", outErr) assert.Contains(t, out, "Success: Submitted") assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress") } @@ -177,12 +178,12 @@ func TestProdSubmitWithJava(t *testing.T) { pkgDir := filepath.Join(t.TempDir(), "app") createApplication(t, pkgDir, true) - httpClient := &mockHttpClient{} + httpClient := &mock.HTTPClient{} httpClient.NextResponse(200, `ok`) execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) - execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"auth", "api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"auth", "cert", pkgDir}}, t, httpClient) // Copy an application package pre-assembled with mvn package testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") @@ -191,7 +192,8 @@ func TestProdSubmitWithJava(t *testing.T) { testZipFile := filepath.Join(testAppDir, "application-test.zip") copyFile(t, filepath.Join(pkgDir, "target", "application-test.zip"), testZipFile) - out, _ := execute(command{homeDir: homeDir, args: []string{"prod", "submit", "-k", filepath.Join(homeDir, "t1.api-key.pem"), pkgDir}}, t, httpClient) + out, outErr := execute(command{homeDir: homeDir, args: []string{"prod", "submit", "-k", filepath.Join(homeDir, "t1.api-key.pem"), pkgDir}}, t, httpClient) + assert.Equal(t, "", outErr) assert.Contains(t, out, "Success: Submitted") assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress") } diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index aef963121aa..d57268b248e 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/mock" ) func TestQuery(t *testing.T) { @@ -19,7 +20,7 @@ func TestQuery(t *testing.T) { } func TestQueryVerbose(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(200, "{\"query\":\"result\"}") cmd := command{args: []string{"query", "-v", "select from sources * where title contains 'foo'"}} out, errOut := execute(cmd, t, client) @@ -54,7 +55,7 @@ func TestServerError(t *testing.T) { } func assertQuery(t *testing.T, expectedQuery string, query ...string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(200, "{\"query\":\"result\"}") assert.Equal(t, "{\n \"query\": \"result\"\n}\n", @@ -62,11 +63,11 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { "query output") queryURL, err := queryServiceURL(client) require.Nil(t, err) - assert.Equal(t, queryURL+"/search/"+expectedQuery, client.lastRequest.URL.String()) + assert.Equal(t, queryURL+"/search/"+expectedQuery, client.LastRequest.URL.String()) } func assertQueryError(t *testing.T, status int, errorMessage string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) assert.Equal(t, @@ -76,7 +77,7 @@ func assertQueryError(t *testing.T, status int, errorMessage string) { } func assertQueryServiceError(t *testing.T, status int, errorMessage string) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextResponse(status, errorMessage) _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) assert.Equal(t, @@ -85,7 +86,7 @@ func assertQueryServiceError(t *testing.T, status int, errorMessage string) { "error output") } -func queryServiceURL(client *mockHttpClient) (string, error) { +func queryServiceURL(client *mock.HTTPClient) (string, error) { service, err := getService("query", 0, "") if err != nil { return "", err diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index f5a846536c5..cbcbb6e5d12 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -54,7 +54,6 @@ Vespa documentation: https://docs.vespa.ai`, colorArg string quietArg bool apiKeyFileArg string - apiKeyArg string stdin io.ReadWriter = os.Stdin color = aurora.NewAurora(false) diff --git a/client/go/cmd/status_test.go b/client/go/cmd/status_test.go index 631aa511459..fe7228697c7 100644 --- a/client/go/cmd/status_test.go +++ b/client/go/cmd/status_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" ) func TestStatusDeployCommand(t *testing.T) { @@ -43,40 +44,40 @@ func TestStatusErrorResponse(t *testing.T) { } func assertDeployStatus(target string, args []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} assert.Equal(t, "Deploy API at "+target+" is ready\n", executeCommand(t, client, []string{"status", "deploy"}, args), "vespa status config-server") - assert.Equal(t, target+"/status.html", client.lastRequest.URL.String()) + assert.Equal(t, target+"/status.html", client.LastRequest.URL.String()) } func assertQueryStatus(target string, args []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} assert.Equal(t, "Container (query API) at "+target+" is ready\n", executeCommand(t, client, []string{"status", "query"}, args), "vespa status container") - assert.Equal(t, target+"/ApplicationStatus", client.lastRequest.URL.String()) + assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) assert.Equal(t, "Container (query API) at "+target+" is ready\n", executeCommand(t, client, []string{"status"}, args), "vespa status (the default)") - assert.Equal(t, target+"/ApplicationStatus", client.lastRequest.URL.String()) + assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) } func assertDocumentStatus(target string, args []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} assert.Equal(t, "Container (document API) at "+target+" is ready\n", executeCommand(t, client, []string{"status", "document"}, args), "vespa status container") - assert.Equal(t, target+"/ApplicationStatus", client.lastRequest.URL.String()) + assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) } func assertQueryStatusError(target string, args []string, t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextStatus(500) cmd := []string{"status", "container"} cmd = append(cmd, args...) diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index 294f98c0f91..d12059a8d12 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -25,7 +25,7 @@ import ( func init() { rootCmd.AddCommand(testCmd) - testCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "dev.aws-us-east-1c", "The zone to use for deployment") + testCmd.PersistentFlags().StringVarP(&zoneArg, zoneFlag, "z", "", "The zone to use for deployment. This defaults to a dev zone") } var testCmd = &cobra.Command{ diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index a5a4c68d93d..1f7d0cff7b2 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -15,12 +15,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" ) func TestSuite(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json") client.NextStatus(200) client.NextStatus(200) @@ -45,7 +46,7 @@ func TestSuite(t *testing.T) { } func TestIllegalFileReference(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextStatus(200) client.NextStatus(200) _, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/illegal-reference.json"}}, t, client) @@ -54,7 +55,7 @@ func TestIllegalFileReference(t *testing.T) { } func TestIllegalRequestUri(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextStatus(200) client.NextStatus(200) _, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/illegal-uri.json"}}, t, client) @@ -63,7 +64,7 @@ func TestIllegalRequestUri(t *testing.T) { } func TestProductionTest(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} client.NextStatus(200) outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/external.json"}}, t, client) assert.Equal(t, "external.json: . OK\n\nSuccess: 1 test OK\n", outBytes) @@ -72,19 +73,19 @@ func TestProductionTest(t *testing.T) { } func TestTestWithoutAssertions(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} _, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client) assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://docs.vespa.ai/en/reference/testing\n", errBytes) } func TestSuiteWithoutTests(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} _, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) assert.Equal(t, "Error: failed to find any tests at testdata/tests/staging-test\nHint: See https://docs.vespa.ai/en/reference/testing\n", errBytes) } func TestSingleTest(t *testing.T) { - client := &mockHttpClient{} + client := &mock.HTTPClient{} searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json") client.NextStatus(200) client.NextStatus(200) @@ -121,7 +122,7 @@ func TestSingleTestWithCloudAndEndpoints(t *testing.T) { ioutil.WriteFile(keyFile, kp.PrivateKey, 0600) ioutil.WriteFile(certFile, kp.Certificate, 0600) - client := &mockHttpClient{} + client := &mock.HTTPClient{} searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json") client.NextStatus(200) client.NextStatus(200) @@ -158,10 +159,10 @@ func createRequest(method string, uri string, body string) *http.Request { } } -func assertRequests(requests []*http.Request, client *mockHttpClient, t *testing.T) { - if assert.Equal(t, len(requests), len(client.requests)) { +func assertRequests(requests []*http.Request, client *mock.HTTPClient, t *testing.T) { + if assert.Equal(t, len(requests), len(client.Requests)) { for i, e := range requests { - a := client.requests[i] + a := client.Requests[i] assert.Equal(t, e.URL.String(), a.URL.String()) assert.Equal(t, e.Method, a.Method) assert.Equal(t, util.ReaderToJSON(e.Body), util.ReaderToJSON(a.Body)) diff --git a/client/go/cmd/version_test.go b/client/go/cmd/version_test.go index 039f75a6ecd..9c05b130e84 100644 --- a/client/go/cmd/version_test.go +++ b/client/go/cmd/version_test.go @@ -6,11 +6,12 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/mock" "github.com/vespa-engine/vespa/client/go/util" ) func TestVersion(t *testing.T) { - c := &mockHttpClient{} + c := &mock.HTTPClient{} c.NextResponse(200, `[{"tag_name": "v1.2.3", "published_at": "2021-09-10T12:00:00Z"}]`) util.ActiveHttpClient = c @@ -21,7 +22,7 @@ func TestVersion(t *testing.T) { } func TestVersionCheckHomebrew(t *testing.T) { - c := &mockHttpClient{} + c := &mock.HTTPClient{} c.NextResponse(200, `[{"tag_name": "v1.2.3", "published_at": "2021-09-10T12:00:00Z"}]`) util.ActiveHttpClient = c diff --git a/client/go/mock/mock.go b/client/go/mock/mock.go new file mode 100644 index 00000000000..f2fcf9c5960 --- /dev/null +++ b/client/go/mock/mock.go @@ -0,0 +1,55 @@ +package mock + +import ( + "bytes" + "crypto/tls" + "io/ioutil" + "net/http" + "strconv" + "time" +) + +type HTTPClient struct { + // The responses to return for future requests. Once a response is consumed, it's removed from this slice + nextResponses []httpResponse + + // LastRequest is the last HTTP request made through this + LastRequest *http.Request + + // Requests contains all requests made through this + Requests []*http.Request +} + +type httpResponse struct { + status int + body []byte +} + +func (c *HTTPClient) NextStatus(status int) { c.NextResponseBytes(status, nil) } + +func (c *HTTPClient) NextResponse(status int, body string) { + c.NextResponseBytes(status, []byte(body)) +} + +func (c *HTTPClient) NextResponseBytes(status int, body []byte) { + c.nextResponses = append(c.nextResponses, httpResponse{status: status, body: body}) +} + +func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { + response := httpResponse{status: 200} + if len(c.nextResponses) > 0 { + response = c.nextResponses[0] + c.nextResponses = c.nextResponses[1:] + } + c.LastRequest = request + c.Requests = append(c.Requests, request) + return &http.Response{ + Status: "Status " + strconv.Itoa(response.status), + StatusCode: response.status, + Body: ioutil.NopCloser(bytes.NewBuffer(response.body)), + Header: make(http.Header), + }, + nil +} + +func (c *HTTPClient) UseCertificate(certificates []tls.Certificate) {} diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index aed430399a0..3316dcac924 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -38,15 +38,15 @@ type ZoneID struct { } type Deployment struct { + System System Application ApplicationID Zone ZoneID } -type DeploymentOpts struct { - ApplicationPackage ApplicationPackage +type DeploymentOptions struct { Target Target - Deployment Deployment - APIKey []byte + ApplicationPackage ApplicationPackage + Timeout time.Duration } type ApplicationPackage struct { @@ -66,13 +66,16 @@ func (d Deployment) String() string { return fmt.Sprintf("deployment of %s in %s", d.Application, d.Zone) } -func (d DeploymentOpts) String() string { - return fmt.Sprintf("%s to %s", d.Deployment, d.Target.Type()) +func (d DeploymentOptions) String() string { + return fmt.Sprintf("%s to %s", d.Target.Deployment(), d.Target.Type()) } -func (d *DeploymentOpts) IsCloud() bool { return d.Target.Type() == cloudTargetType } +// IsCloud returns whether this is a deployment to Vespa Cloud or hosted Vespa +func (d *DeploymentOptions) IsCloud() bool { + return d.Target.Type() == TargetCloud || d.Target.Type() == TargetHosted +} -func (d *DeploymentOpts) url(path string) (*url.URL, error) { +func (d *DeploymentOptions) url(path string) (*url.URL, error) { service, err := d.Target.Service(deployService, 0, 0, "") if err != nil { return nil, err @@ -196,7 +199,7 @@ func ZoneFromString(s string) (ZoneID, error) { } // Prepare deployment and return the session ID -func Prepare(deployment DeploymentOpts) (int64, error) { +func Prepare(deployment DeploymentOptions) (int64, error) { if deployment.IsCloud() { return 0, fmt.Errorf("prepare is not supported with %s target", deployment.Target.Type()) } @@ -229,7 +232,7 @@ func Prepare(deployment DeploymentOpts) (int64, error) { } // Activate deployment with sessionID from a past prepare -func Activate(sessionID int64, deployment DeploymentOpts) error { +func Activate(sessionID int64, deployment DeploymentOptions) error { if deployment.IsCloud() { return fmt.Errorf("activate is not supported with %s target", deployment.Target.Type()) } @@ -250,21 +253,21 @@ func Activate(sessionID int64, deployment DeploymentOpts) error { return checkResponse(req, response, serviceDescription) } -func Deploy(opts DeploymentOpts) (int64, error) { +func Deploy(opts DeploymentOptions) (int64, error) { path := "/application/v2/tenant/default/prepareandactivate" if opts.IsCloud() { if err := checkDeploymentOpts(opts); err != nil { return 0, err } - if opts.Deployment.Zone.Environment == "" || opts.Deployment.Zone.Region == "" { + if opts.Target.Deployment().Zone.Environment == "" || opts.Target.Deployment().Zone.Region == "" { return 0, fmt.Errorf("%s: missing zone", opts) } path = fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/deploy/%s-%s", - opts.Deployment.Application.Tenant, - opts.Deployment.Application.Application, - opts.Deployment.Application.Instance, - opts.Deployment.Zone.Environment, - opts.Deployment.Zone.Region) + opts.Target.Deployment().Application.Tenant, + opts.Target.Deployment().Application.Application, + opts.Target.Deployment().Application.Instance, + opts.Target.Deployment().Zone.Environment, + opts.Target.Deployment().Zone.Region) } u, err := opts.url(path) if err != nil { @@ -290,14 +293,14 @@ func copyToPart(dst *multipart.Writer, src io.Reader, fieldname, filename string return nil } -func Submit(opts DeploymentOpts) error { +func Submit(opts DeploymentOptions) error { if !opts.IsCloud() { - return fmt.Errorf("%s: submit is unsupported", opts) + return fmt.Errorf("%s: submit is unsupported by %s target", opts, opts.Target.Type()) } if err := checkDeploymentOpts(opts); err != nil { return err } - path := fmt.Sprintf("/application/v4/tenant/%s/application/%s/submit", opts.Deployment.Application.Tenant, opts.Deployment.Application.Application) + path := fmt.Sprintf("/application/v4/tenant/%s/application/%s/submit", opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application) u, err := opts.url(path) if err != nil { return err @@ -332,7 +335,7 @@ func Submit(opts DeploymentOpts) error { } request.Header.Set("Content-Type", writer.FormDataContentType()) serviceDescription := "Submit service" - sigKeyId := opts.Deployment.Application.SerializedForm() + sigKeyId := opts.Target.Deployment().Application.SerializedForm() if err := opts.Target.SignRequest(request, sigKeyId); err != nil { return err } @@ -344,14 +347,14 @@ func Submit(opts DeploymentOpts) error { return checkResponse(request, response, serviceDescription) } -func checkDeploymentOpts(opts DeploymentOpts) error { - if !opts.ApplicationPackage.HasCertificate() { +func checkDeploymentOpts(opts DeploymentOptions) error { + if opts.Target.Type() == TargetCloud && !opts.ApplicationPackage.HasCertificate() { return fmt.Errorf("%s: missing certificate in package", opts) } return nil } -func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) { +func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (int64, error) { zipReader, err := opts.ApplicationPackage.zipReader(false) if err != nil { return 0, err @@ -364,15 +367,18 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) Header: header, Body: ioutil.NopCloser(zipReader), } - serviceDescription := "Deploy service" - sigKeyId := opts.Deployment.Application.SerializedForm() - if err := opts.Target.SignRequest(request, sigKeyId); err != nil { + service, err := opts.Target.Service(deployService, opts.Timeout, 0, "") + if err != nil { return 0, err } + keyID := opts.Target.Deployment().Application.SerializedForm() + if err := opts.Target.SignRequest(request, keyID); err != nil { + return 0, err + } var response *http.Response err = util.Spinner("Uploading application package ...", func() error { - response, err = util.HttpDo(request, time.Minute*10, serviceDescription) + response, err = service.Do(request, time.Minute*10) return err }) if err != nil { @@ -385,7 +391,7 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOpts) (int64, error) RunID int64 `json:"run"` // Controller } jsonResponse.SessionID = "0" // Set a default session ID for responses that don't contain int (e.g. cloud deployment) - if err := checkResponse(request, response, serviceDescription); err != nil { + if err := checkResponse(request, response, service.Description()); err != nil { return 0, err } jsonDec := json.NewDecoder(response.Body) diff --git a/client/go/vespa/system.go b/client/go/vespa/system.go new file mode 100644 index 00000000000..a6cf1a9d9ff --- /dev/null +++ b/client/go/vespa/system.go @@ -0,0 +1,68 @@ +package vespa + +import "fmt" + +// PublicSystem represents the main Vespa Cloud system. +var PublicSystem = System{ + Name: "public", + URL: "https://api.vespa-external.aws.oath.cloud:4443", + ConsoleURL: "https://console.vespa.oath.cloud", + DefaultZone: ZoneID{Environment: "dev", Region: "aws-us-east-1c"}, +} + +// PublicCDSystem represents the CD variant of the Vespa Cloud system. +var PublicCDSystem = System{ + Name: "publiccd", + URL: "https://api.vespa-external-cd.aws.oath.cloud:4443", + ConsoleURL: "https://console-cd.vespa.oath.cloud", + DefaultZone: ZoneID{Environment: "dev", Region: "aws-us-east-1c"}, +} + +// MainSystem represents the main hosted Vespa system. +var MainSystem = System{ + Name: "main", + URL: "https://api.vespa.ouryahoo.com:4443", + ConsoleURL: "https://console.vespa.ouryahoo.com", + DefaultZone: ZoneID{Environment: "dev", Region: "us-east-1"}, + AthenzDomain: "vespa.vespa", +} + +// CDSystem represents the CD variant of the hosted Vespa system. +var CDSystem = System{ + Name: "cd", + URL: "https://api-cd.vespa.ouryahoo.com:4443", + ConsoleURL: "https://console-cd.vespa.ouryahoo.com", + DefaultZone: ZoneID{Environment: "dev", Region: "cd-us-west-1"}, + AthenzDomain: "vespa.vespa.cd", +} + +// System represents a Vespa system. +type System struct { + Name string + // URL is the API URL for this system. + URL string + ConsoleURL string + // DefaultZone is default zone to use in manual deployments to this system. + DefaultZone ZoneID + // AthenzDomain is the Athenz domain used by this system. This is empty for systems not using Athenz for tenant + // authentication. + AthenzDomain string +} + +// IsPublic returns whether system s is a public (Vespa Cloud) system. +func (s *System) IsPublic() bool { return s.Name == PublicSystem.Name || s.Name == PublicCDSystem.Name } + +// GetSystem returns the system of given name. +func GetSystem(name string) (System, error) { + switch name { + case "cd": + return CDSystem, nil + case "main": + return MainSystem, nil + case "public": + return PublicSystem, nil + case "publiccd": + return PublicCDSystem, nil + } + return System{}, fmt.Errorf("invalid system: %s", name) +} diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 204dbc143c6..f620f3b865c 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -19,12 +19,21 @@ import ( "github.com/vespa-engine/vespa/client/go/auth0" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/version" + "github.com/vespa-engine/vespa/client/go/zts" ) const ( - localTargetType = "local" - customTargetType = "custom" - cloudTargetType = "cloud" + // A target for a local Vespa service + TargetLocal = "local" + + // A target for a custom URL + TargetCustom = "custom" + + // A Vespa Cloud target + TargetCloud = "cloud" + + // A hosted Vespa target + TargetHosted = "hosted" deployService = "deploy" queryService = "query" @@ -33,16 +42,12 @@ const ( retryInterval = 2 * time.Second ) -const ( - CloudAuthApiKey = "api-key" - CloudAuthAccessToken = "access-token" -) - // Service represents a Vespa service. type Service struct { BaseURL string Name string TLSOptions TLSOptions + ztsClient ztsClient } // Target represents a Vespa platform, running named Vespa services. @@ -50,6 +55,9 @@ type Target interface { // Type returns this target's type, e.g. local or cloud. Type() string + // Deployment returns the deployment managed by this target. + Deployment() Deployment + // Service returns the service for given name. If timeout is non-zero, wait for the service to converge. Service(name string, timeout time.Duration, sessionOrRunID int64, cluster string) (*Service, error) @@ -63,11 +71,12 @@ type Target interface { CheckVersion(clientVersion version.Version) error } -// TLSOptions configures the certificate to use for service requests. +// TLSOptions configures the client certificate to use for cloud API or service requests. type TLSOptions struct { KeyPair tls.Certificate CertificateFile string PrivateKeyFile string + AthenzDomain string } // LogOptions configures the log output to produce when writing log messages. @@ -80,20 +89,41 @@ type LogOptions struct { Level int } +// CloudOptions configures URL and authentication for a cloud target. +type APIOptions struct { + System System + TLSOptions TLSOptions + APIKey []byte + AuthConfigPath string +} + +// CloudDeploymentOptions configures the deployment to manage through a cloud target. +type CloudDeploymentOptions struct { + Deployment Deployment + TLSOptions TLSOptions + ClusterURLs map[string]string // Endpoints keyed on cluster name +} + type customTarget struct { targetType string baseURL string } -func (t *customTarget) SignRequest(req *http.Request, sigKeyId string) error { return nil } - -func (t *customTarget) CheckVersion(version version.Version) error { return nil } - // Do sends request to this service. Any required authentication happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if s.TLSOptions.KeyPair.Certificate != nil { util.ActiveHttpClient.UseCertificate([]tls.Certificate{s.TLSOptions.KeyPair}) } + if s.TLSOptions.AthenzDomain != "" { + accessToken, err := s.ztsClient.AccessToken(s.TLSOptions.AthenzDomain, s.TLSOptions.KeyPair) + if err != nil { + return nil, err + } + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Add("Authorization", "Bearer "+accessToken) + } return util.HttpDo(request, timeout, s.Description()) } @@ -130,6 +160,8 @@ func (s *Service) Description() string { func (t *customTarget) Type() string { return t.targetType } +func (t *customTarget) Deployment() Deployment { return Deployment{} } + func (t *customTarget) Service(name string, timeout time.Duration, sessionOrRunID int64, cluster string) (*Service, error) { if timeout > 0 && name != deployService { if err := t.waitForConvergence(timeout); err != nil { @@ -148,9 +180,13 @@ func (t *customTarget) Service(name string, timeout time.Duration, sessionOrRunI } func (t *customTarget) PrintLog(options LogOptions) error { - return fmt.Errorf("reading logs from non-cloud deployment is currently unsupported") + return fmt.Errorf("reading logs from non-cloud deployment is unsupported") } +func (t *customTarget) SignRequest(req *http.Request, sigKeyId string) error { return nil } + +func (t *customTarget) CheckVersion(version version.Version) error { return nil } + func (t *customTarget) urlWithPort(serviceName string) (string, error) { u, err := url.Parse(t.baseURL) if err != nil { @@ -203,32 +239,30 @@ func (t *customTarget) waitForConvergence(timeout time.Duration) error { } type cloudTarget struct { - apiURL string - targetType string - deployment Deployment - apiKey []byte - tlsOptions TLSOptions - logOptions LogOptions + apiOptions APIOptions + deploymentOptions CloudDeploymentOptions + logOptions LogOptions + ztsClient ztsClient +} - urlsByCluster map[string]string - authConfigPath string - systemName string +type ztsClient interface { + AccessToken(domain string, certficiate tls.Certificate) (string, error) } func (t *cloudTarget) resolveEndpoint(cluster string) (string, error) { if cluster == "" { - for _, u := range t.urlsByCluster { - if len(t.urlsByCluster) == 1 { + for _, u := range t.deploymentOptions.ClusterURLs { + if len(t.deploymentOptions.ClusterURLs) == 1 { return u, nil } else { - return "", fmt.Errorf("multiple clusters, none chosen: %v", t.urlsByCluster) + return "", fmt.Errorf("multiple clusters, none chosen: %v", t.deploymentOptions.ClusterURLs) } } } else { - u := t.urlsByCluster[cluster] + u := t.deploymentOptions.ClusterURLs[cluster] if u == "" { - clusters := make([]string, len(t.urlsByCluster)) - for c := range t.urlsByCluster { + clusters := make([]string, len(t.deploymentOptions.ClusterURLs)) + for c := range t.deploymentOptions.ClusterURLs { clusters = append(clusters, c) } return "", fmt.Errorf("unknown cluster '%s': must be one of %v", cluster, clusters) @@ -239,54 +273,57 @@ func (t *cloudTarget) resolveEndpoint(cluster string) (string, error) { return "", fmt.Errorf("no endpoints") } -func (t *cloudTarget) Type() string { return t.targetType } +func (t *cloudTarget) Type() string { + switch t.apiOptions.System.Name { + case MainSystem.Name, CDSystem.Name: + return TargetHosted + } + return TargetCloud +} + +func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment } func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) { - if name != deployService && t.urlsByCluster == nil { + if name != deployService && t.deploymentOptions.ClusterURLs == nil { if err := t.waitForEndpoints(timeout, runID); err != nil { return nil, err } } switch name { case deployService: - return &Service{Name: name, BaseURL: t.apiURL}, nil - case queryService: - queryURL, err := t.resolveEndpoint(cluster) - if err != nil { - return nil, err - } - return &Service{Name: name, BaseURL: queryURL, TLSOptions: t.tlsOptions}, nil - case documentService: - documentURL, err := t.resolveEndpoint(cluster) + return &Service{Name: name, BaseURL: t.apiOptions.System.URL, TLSOptions: t.apiOptions.TLSOptions, ztsClient: t.ztsClient}, nil + case queryService, documentService: + url, err := t.resolveEndpoint(cluster) if err != nil { return nil, err } - return &Service{Name: name, BaseURL: documentURL, TLSOptions: t.tlsOptions}, nil + t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain + return &Service{Name: name, BaseURL: url, TLSOptions: t.deploymentOptions.TLSOptions, ztsClient: t.ztsClient}, nil } return nil, fmt.Errorf("unknown service: %s", name) } -// SignRequest adds authentication data to a http.Request. -// The api key is used if set on cloudTarget, if not the Auth0 device flow is used. -func (t *cloudTarget) SignRequest(req *http.Request, sigKeyId string) error { - if t.apiKey != nil { - signer := NewRequestSigner(sigKeyId, t.apiKey) - if err := signer.SignRequest(req); err != nil { - return err +func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error { + if t.apiOptions.System.IsPublic() { + if t.apiOptions.APIKey != nil { + signer := NewRequestSigner(keyID, t.apiOptions.APIKey) + return signer.SignRequest(req) + } else { + return t.addAuth0AccessToken(req) } } else { - if err := t.addAuth0AccessToken(req); err != nil { - return err + if t.apiOptions.TLSOptions.KeyPair.Certificate == nil { + return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name) } + return nil } - return nil } func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { if clientVersion.IsZero() { // development version is always fine return nil } - req, err := http.NewRequest("GET", fmt.Sprintf("%s/cli/v1/", t.apiURL), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("%s/cli/v1/", t.apiOptions.System.URL), nil) if err != nil { return err } @@ -313,7 +350,7 @@ func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { } func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { - a, err := auth0.GetAuth0(t.authConfigPath, t.systemName, t.apiURL) + a, err := auth0.GetAuth0(t.apiOptions.AuthConfigPath, t.apiOptions.System.Name, t.apiOptions.System.URL) if err != nil { return err } @@ -327,9 +364,9 @@ func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { func (t *cloudTarget) logsURL() string { return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs", - t.apiURL, - t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, - t.deployment.Zone.Environment, t.deployment.Zone.Region) + t.apiOptions.System.URL, + t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, + t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region) } func (t *cloudTarget) PrintLog(options LogOptions) error { @@ -347,7 +384,7 @@ func (t *cloudTarget) PrintLog(options LogOptions) error { q.Set("to", strconv.FormatInt(toMillis, 10)) } req.URL.RawQuery = q.Encode() - t.SignRequest(req, t.deployment.Application.SerializedForm()) + t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()) return req } logFunc := func(status int, response []byte) (bool, error) { @@ -376,7 +413,7 @@ func (t *cloudTarget) PrintLog(options LogOptions) error { if options.Follow { timeout = math.MaxInt64 // No timeout } - _, err = wait(logFunc, requestFunc, &t.tlsOptions.KeyPair, timeout) + _, err = wait(logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) return err } @@ -391,9 +428,9 @@ func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d", - t.apiURL, - t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, - t.deployment.Zone.Environment, t.deployment.Zone.Region, runID) + t.apiOptions.System.URL, + t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, + t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region, runID) req, err := http.NewRequest("GET", runURL, nil) if err != nil { return err @@ -403,7 +440,7 @@ func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { q := req.URL.Query() q.Set("after", strconv.FormatInt(lastID, 10)) req.URL.RawQuery = q.Encode() - if err := t.SignRequest(req, t.deployment.Application.SerializedForm()); err != nil { + if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { panic(err) } return req @@ -427,7 +464,7 @@ func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { } return true, nil } - _, err = wait(jobSuccessFunc, requestFunc, &t.tlsOptions.KeyPair, timeout) + _, err = wait(jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) return err } @@ -455,14 +492,14 @@ func (t *cloudTarget) printLog(response jobResponse, last int64) int64 { func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s", - t.apiURL, - t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, - t.deployment.Zone.Environment, t.deployment.Zone.Region) + t.apiOptions.System.URL, + t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, + t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region) req, err := http.NewRequest("GET", deploymentURL, nil) if err != nil { return err } - if err := t.SignRequest(req, t.deployment.Application.SerializedForm()); err != nil { + if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { return err } urlsByCluster := make(map[string]string) @@ -485,13 +522,13 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { } return true, nil } - if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.tlsOptions.KeyPair, timeout); err != nil { + if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil { return err } if len(urlsByCluster) == 0 { return fmt.Errorf("no endpoints discovered") } - t.urlsByCluster = urlsByCluster + t.deploymentOptions.ClusterURLs = urlsByCluster return nil } @@ -504,28 +541,26 @@ func isOK(status int) (bool, error) { // LocalTarget creates a target for a Vespa platform running locally. func LocalTarget() Target { - return &customTarget{targetType: localTargetType, baseURL: "http://127.0.0.1"} + return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1"} } // CustomTarget creates a Target for a Vespa platform running at baseURL. func CustomTarget(baseURL string) Target { - return &customTarget{targetType: customTargetType, baseURL: baseURL} + return &customTarget{targetType: TargetCustom, baseURL: baseURL} } -// CloudTarget creates a Target for the Vespa Cloud platform. -func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions, - authConfigPath string, systemName string, urlsByCluster map[string]string) Target { - return &cloudTarget{ - apiURL: apiURL, - targetType: cloudTargetType, - deployment: deployment, - apiKey: apiKey, - tlsOptions: tlsOptions, - logOptions: logOptions, - authConfigPath: authConfigPath, - systemName: systemName, - urlsByCluster: urlsByCluster, +// CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform. +func CloudTarget(apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { + ztsClient, err := zts.NewClient(zts.DefaultURL, util.ActiveHttpClient) + if err != nil { + return nil, err } + return &cloudTarget{ + apiOptions: apiOptions, + deploymentOptions: deploymentOptions, + logOptions: logOptions, + ztsClient: ztsClient, + }, nil } type deploymentEndpoint struct { @@ -571,7 +606,8 @@ func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, time deadline := time.Now().Add(timeout) loopOnce := timeout == 0 for time.Now().Before(deadline) || loopOnce { - response, httpErr = util.HttpDo(reqFn(), 10*time.Second, "") + req := reqFn() + response, httpErr = util.HttpDo(req, 10*time.Second, "") if httpErr == nil { statusCode = response.StatusCode body, err := ioutil.ReadAll(response.Body) diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index 8391655eaf7..bf3e0fae7d0 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -169,12 +169,23 @@ func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { apiKey, err := CreateAPIKey() assert.Nil(t, err) - target := CloudTarget("https://example.com", Deployment{ - Application: ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, - Zone: ZoneID{Environment: "dev", Region: "us-north-1"}, - }, apiKey, TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: logWriter}, "", "", nil) + target, err := CloudTarget( + APIOptions{APIKey: apiKey, System: PublicSystem}, + CloudDeploymentOptions{ + Deployment: Deployment{ + Application: ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, + Zone: ZoneID{Environment: "dev", Region: "us-north-1"}, + }, + TLSOptions: TLSOptions{KeyPair: x509KeyPair}, + }, + LogOptions{Writer: logWriter}, + ) + if err != nil { + t.Fatal(err) + } if ct, ok := target.(*cloudTarget); ok { - ct.apiURL = url + ct.apiOptions.System.URL = url + ct.ztsClient = &mockZTSClient{token: "foo bar"} } else { t.Fatalf("Wrong target type %T", ct) } @@ -195,3 +206,11 @@ func assertServiceWait(t *testing.T, expectedStatus int, target Target, service assert.Nil(t, err) assert.Equal(t, expectedStatus, status) } + +type mockZTSClient struct { + token string +} + +func (c *mockZTSClient) AccessToken(domain string, certificate tls.Certificate) (string, error) { + return c.token, nil +} diff --git a/client/go/vespa/xml/config.go b/client/go/vespa/xml/config.go index c9efcb7f340..c9af92339bc 100644 --- a/client/go/vespa/xml/config.go +++ b/client/go/vespa/xml/config.go @@ -9,6 +9,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/vespa-engine/vespa/client/go/vespa" ) var DefaultDeployment Deployment @@ -218,8 +220,9 @@ func ParseNodeCount(s string) (int, int, error) { } // IsProdRegion returns whether string s is a valid production region. -func IsProdRegion(s string, system string) bool { - if system == "publiccd" { +func IsProdRegion(s string, system vespa.System) bool { + // TODO: Add support for cd and main systems + if system.Name == vespa.PublicCDSystem.Name { return s == "aws-us-east-1c" } switch s { diff --git a/client/go/zts/zts.go b/client/go/zts/zts.go new file mode 100644 index 00000000000..b1a47db8e48 --- /dev/null +++ b/client/go/zts/zts.go @@ -0,0 +1,55 @@ +package zts + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/vespa-engine/vespa/client/go/util" +) + +const DefaultURL = "https://zts.athenz.ouroath.com:4443" + +// Client is a client for Athenz ZTS, an authentication token service. +type Client struct { + client util.HttpClient + tokenURL *url.URL +} + +// NewClient creates a new client for an Athenz ZTS service located at serviceURL. +func NewClient(serviceURL string, client util.HttpClient) (*Client, error) { + tokenURL, err := url.Parse(serviceURL) + if err != nil { + return nil, err + } + tokenURL.Path = "/zts/v1/oauth2/token" + return &Client{tokenURL: tokenURL, client: client}, nil +} + +// AccessToken returns an access token within the given domain, using certificate to authenticate with ZTS. +func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string, error) { + data := fmt.Sprintf("grant_type=client_credentials&scope=%s:domain", domain) + req, err := http.NewRequest("POST", c.tokenURL.String(), strings.NewReader(data)) + if err != nil { + return "", err + } + c.client.UseCertificate([]tls.Certificate{certificate}) + response, err := c.client.Do(req, 10*time.Second) + if err != nil { + return "", err + } + defer response.Body.Close() + + var ztsResponse struct { + AccessToken string `json:"access_token"` + } + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&ztsResponse); err != nil { + return "", err + } + return ztsResponse.AccessToken, nil +} diff --git a/client/go/zts/zts_test.go b/client/go/zts/zts_test.go new file mode 100644 index 00000000000..f1bd9c1ba75 --- /dev/null +++ b/client/go/zts/zts_test.go @@ -0,0 +1,25 @@ +package zts + +import ( + "crypto/tls" + "testing" + + "github.com/vespa-engine/vespa/client/go/mock" +) + +func TestAccessToken(t *testing.T) { + httpClient := mock.HTTPClient{} + client, err := NewClient("http://example.com", &httpClient) + if err != nil { + t.Fatal(err) + } + httpClient.NextResponse(200, `{"access_token": "foo bar"}`) + token, err := client.AccessToken("vespa.vespa", tls.Certificate{}) + if err != nil { + t.Fatal(err) + } + want := "foo bar" + if token != want { + t.Errorf("got %q, want %q", token, want) + } +} |