diff options
23 files changed, 210 insertions, 119 deletions
diff --git a/client/go/go.mod b/client/go/go.mod index f5f923cc063..5d1f6175e55 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -1,13 +1,13 @@ module github.com/vespa-engine/vespa/client/go -go 1.18 +go 1.19 require ( github.com/alessio/shellescape v1.4.1 github.com/briandowns/spinner v1.23.0 github.com/fatih/color v1.15.0 - // This is the most recent version compatible with Go 1.18. Upgrade when we upgrade our Go version - github.com/go-json-experiment/json v0.0.0-20220727223814-4987ed27d447 + // This is the most recent version compatible with Go 1.19. Upgrade when we upgrade our Go version + github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424 github.com/klauspost/compress v1.16.5 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.18 diff --git a/client/go/go.sum b/client/go/go.sum index 861c8725ed0..03206b0c5e8 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -11,8 +11,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/go-json-experiment/json v0.0.0-20220727223814-4987ed27d447 h1:hDdASyrtiSuQvaafDrVTX34wy4ibhxrJO9/vyFbBt0k= -github.com/go-json-experiment/json v0.0.0-20220727223814-4987ed27d447/go.mod h1:jbpkervfdK2HCcB2YEFmwYeaq057KFiaaKTNTHV4OOQ= +github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424 h1:I1EK0t+BDH+kvlozNqrvzKqsWeM2QUKxXH0iW2fjDDw= +github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424/go.mod h1:I+I5/LT2lLP0eZsBNaVDrOrYASx9h7o7mRHmy+535/A= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/client/go/internal/cli/cmd/api_key.go b/client/go/internal/cli/cmd/api_key.go index 367a515f3c3..8b3780ab82b 100644 --- a/client/go/internal/cli/cmd/api_key.go +++ b/client/go/internal/cli/cmd/api_key.go @@ -58,11 +58,11 @@ func doApiKey(cli *CLI, overwriteKey bool, args []string) error { if err != nil { return err } - targetType, err := cli.config.targetType() + targetType, err := cli.targetType() if err != nil { return err } - system, err := cli.system(targetType) + system, err := cli.system(targetType.name) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index 48bad974c3f..95206b7e77d 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -107,15 +107,15 @@ func doCert(cli *CLI, overwriteCertificate, noApplicationPackage bool, args []st return err } } - targetType, err := cli.config.targetType() + targetType, err := cli.targetType() if err != nil { return err } - privateKeyFile, err := cli.config.privateKeyPath(app, targetType) + privateKeyFile, err := cli.config.privateKeyPath(app, targetType.name) if err != nil { return err } - certificateFile, err := cli.config.certificatePath(app, targetType) + certificateFile, err := cli.config.certificatePath(app, targetType.name) if err != nil { return err } @@ -178,11 +178,11 @@ func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { if err != nil { return err } - targetType, err := cli.config.targetType() + targetType, err := cli.targetType() if err != nil { return err } - certificateFile, err := cli.config.certificatePath(app, targetType) + certificateFile, err := cli.config.certificatePath(app, targetType.name) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index e2132814386..0e120546c8b 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -329,7 +329,7 @@ func (c *Config) write() error { return c.config.WriteFile(configFile) } -func (c *Config) targetType() (string, error) { +func (c *Config) targetOrURL() (string, error) { targetType, ok := c.get(targetFlag) if !ok { return "", fmt.Errorf("target is unset") diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go index 66b65bf402b..3a81b93ea0d 100644 --- a/client/go/internal/cli/cmd/config_test.go +++ b/client/go/internal/cli/cmd/config_test.go @@ -261,6 +261,22 @@ func TestConfigReadTLSOptions(t *testing.T) { ) } +func TestConfigTargetResolving(t *testing.T) { + cli, _, _ := newTestCLI(t) + require.Nil(t, cli.Run("config", "set", "target", "https://example.com")) + assertTargetType(t, vespa.TargetCustom, cli) + require.Nil(t, cli.Run("config", "set", "target", "https://foo.bar.vespa-team.no-north-1.dev.z.vespa-app.cloud")) + assertTargetType(t, vespa.TargetCloud, cli) + require.Nil(t, cli.Run("config", "set", "target", "https://foo.bar.vespa-team.no-north-1.dev.z.vespa.oath.cloud:4443")) + assertTargetType(t, vespa.TargetHosted, cli) +} + +func assertTargetType(t *testing.T, expected string, cli *CLI) { + targetType, err := cli.targetType() + require.Nil(t, err) + assert.Equal(t, expected, targetType.name) +} + func assertTLSOptions(t *testing.T, homeDir string, app vespa.ApplicationID, target string, want vespa.TLSOptions, envVars ...string) { t.Helper() envVars = append(envVars, "VESPA_CLI_HOME="+homeDir) diff --git a/client/go/internal/cli/cmd/login.go b/client/go/internal/cli/cmd/login.go index 9ac2262e78d..d2075bdfcf0 100644 --- a/client/go/internal/cli/cmd/login.go +++ b/client/go/internal/cli/cmd/login.go @@ -27,11 +27,11 @@ func newLoginCmd(cli *CLI) *cobra.Command { SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - targetType, err := cli.config.targetType() + targetType, err := cli.targetType() if err != nil { return err } - system, err := cli.system(targetType) + system, err := cli.system(targetType.name) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/logout.go b/client/go/internal/cli/cmd/logout.go index 32e7cd9783b..93f7cb6270f 100644 --- a/client/go/internal/cli/cmd/logout.go +++ b/client/go/internal/cli/cmd/logout.go @@ -14,11 +14,11 @@ func newLogoutCmd(cli *CLI) *cobra.Command { DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - targetType, err := cli.config.targetType() + targetType, err := cli.targetType() if err != nil { return err } - system, err := cli.system(targetType) + system, err := cli.system(targetType.name) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index c4012024426..17c4fc41625 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "net/url" "os" "os/exec" "strings" @@ -73,6 +74,11 @@ type targetOptions struct { noCertificate bool } +type targetType struct { + name string + url string +} + // errHint creates a new CLI error, with optional hints that will be printed after the error func errHint(err error, hints ...string) ErrCLI { return ErrCLI{Status: 1, hints: hints, error: err} } @@ -297,7 +303,19 @@ func (c *CLI) printWarning(msg interface{}, hints ...string) { // target creates a target according the configuration of this CLI and given opts. func (c *CLI) target(opts targetOptions) (vespa.Target, error) { - target, err := c.createTarget(opts) + targetType, err := c.targetType() + if err != nil { + return nil, err + } + var target vespa.Target + switch targetType.name { + case vespa.TargetLocal, vespa.TargetCustom: + target, err = c.createCustomTarget(targetType.name, targetType.url) + case vespa.TargetCloud, vespa.TargetHosted: + target, err = c.createCloudTarget(targetType.name, opts, targetType.url) + default: + return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") + } if err != nil { return nil, err } @@ -309,24 +327,39 @@ func (c *CLI) target(opts targetOptions) (vespa.Target, error) { return target, nil } -func (c *CLI) createTarget(opts targetOptions) (vespa.Target, error) { - targetType, err := c.config.targetType() +// targetType resolves the real target type and its custom URL (if any) +func (c *CLI) targetType() (targetType, error) { + v, err := c.config.targetOrURL() if err != nil { - return nil, err + return targetType{}, err } - customURL := "" - if strings.HasPrefix(targetType, "http") { - customURL = targetType - targetType = vespa.TargetCustom + tt := targetType{name: v} + if strings.HasPrefix(tt.name, "http://") || strings.HasPrefix(tt.name, "https://") { + tt.url = tt.name + tt.name, err = c.targetFromURL(tt.url) + if err != nil { + return targetType{}, err + } } - switch targetType { - case vespa.TargetLocal, vespa.TargetCustom: - return c.createCustomTarget(targetType, customURL) - case vespa.TargetCloud, vespa.TargetHosted: - return c.createCloudTarget(targetType, opts) - default: - return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") + return tt, nil +} + +func (c *CLI) targetFromURL(customURL string) (string, error) { + u, err := url.Parse(customURL) + if err != nil { + return "", err + } + // Check if URL belongs to a cloud target + for _, cloudTarget := range []string{vespa.TargetHosted, vespa.TargetCloud} { + system, err := c.system(cloudTarget) + if err != nil { + return "", err + } + if strings.HasSuffix(u.Hostname(), "."+system.EndpointDomain) { + return cloudTarget, nil + } } + return vespa.TargetCustom, nil } func (c *CLI) createCustomTarget(targetType, customURL string) (vespa.Target, error) { @@ -344,7 +377,7 @@ func (c *CLI) createCustomTarget(targetType, customURL string) (vespa.Target, er } } -func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Target, error) { +func (c *CLI) createCloudTarget(targetType string, opts targetOptions, customURL string) (vespa.Target, error) { system, err := c.system(targetType) if err != nil { return nil, err @@ -409,6 +442,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta deploymentOptions := vespa.CloudDeploymentOptions{ Deployment: deployment, TLSOptions: deploymentTLSOptions, + CustomURL: customURL, ClusterURLs: endpoints, } logLevel := opts.logLevel diff --git a/client/go/internal/cli/cmd/status_test.go b/client/go/internal/cli/cmd/status_test.go index a3cae7c3fe4..76efea55503 100644 --- a/client/go/internal/cli/cmd/status_test.go +++ b/client/go/internal/cli/cmd/status_test.go @@ -16,7 +16,7 @@ func TestStatusDeployCommand(t *testing.T) { } func TestStatusDeployCommandWithURLTarget(t *testing.T) { - assertDeployStatus("http://mydeploytarget:19071", []string{"-t", "http://mydeploytarget"}, t) + assertDeployStatus("http://mydeploytarget:19071", []string{"-t", "http://mydeploytarget:19071"}, t) } func TestStatusDeployCommandWithLocalTarget(t *testing.T) { @@ -28,7 +28,7 @@ func TestStatusQueryCommand(t *testing.T) { } func TestStatusQueryCommandWithUrlTarget(t *testing.T) { - assertQueryStatus("http://mycontainertarget:8080", []string{"-t", "http://mycontainertarget"}, t) + assertQueryStatus("http://mycontainertarget:8080", []string{"-t", "http://mycontainertarget:8080"}, t) } func TestStatusQueryCommandWithLocalTarget(t *testing.T) { diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go index 7237a87b7e2..2ad5b841616 100644 --- a/client/go/internal/vespa/document/dispatcher.go +++ b/client/go/internal/vespa/document/dispatcher.go @@ -25,7 +25,7 @@ type Dispatcher struct { msgs chan string inflight map[string]*Queue[documentOp] - inflightCount int64 + inflightCount atomic.Int64 output io.Writer verbose bool @@ -76,7 +76,7 @@ func (d *Dispatcher) shouldRetry(op documentOp, result Result) bool { } if result.HTTPStatus == 429 || result.HTTPStatus == 503 { d.msgs <- fmt.Sprintf("feed: %s was throttled with status %d: retrying", op.document, result.HTTPStatus) - d.throttler.Throttled(atomic.LoadInt64(&d.inflightCount)) + d.throttler.Throttled(d.inflightCount.Load()) return true } if result.Err != nil || result.HTTPStatus == 500 || result.HTTPStatus == 502 || result.HTTPStatus == 504 { @@ -226,20 +226,20 @@ func (d *Dispatcher) acceptDocument() bool { } func (d *Dispatcher) acquireSlot() { - for atomic.LoadInt64(&d.inflightCount) >= d.throttler.TargetInflight() { + for d.inflightCount.Load() >= d.throttler.TargetInflight() { time.Sleep(time.Millisecond) } - atomic.AddInt64(&d.inflightCount, 1) + d.inflightCount.Add(1) } -func (d *Dispatcher) releaseSlot() { atomic.AddInt64(&d.inflightCount, -1) } +func (d *Dispatcher) releaseSlot() { d.inflightCount.Add(-1) } func (d *Dispatcher) Enqueue(doc Document) error { return d.enqueue(documentOp{document: doc}, false) } func (d *Dispatcher) Stats() Stats { d.statsMu.Lock() defer d.statsMu.Unlock() - d.stats.Inflight = atomic.LoadInt64(&d.inflightCount) + d.stats.Inflight = d.inflightCount.Load() return d.stats } diff --git a/client/go/internal/vespa/document/throttler.go b/client/go/internal/vespa/document/throttler.go index 667a10d28e3..e32fb804b23 100644 --- a/client/go/internal/vespa/document/throttler.go +++ b/client/go/internal/vespa/document/throttler.go @@ -23,11 +23,11 @@ type Throttler interface { type dynamicThrottler struct { minInflight int64 maxInflight int64 - targetInflight int64 - targetTimesTen int64 + targetInflight atomic.Int64 + targetTimesTen atomic.Int64 throughputs []float64 - ok int64 + ok atomic.Int64 sent int64 start time.Time @@ -39,23 +39,24 @@ func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler { minInflight = 16 * int64(connections) maxInflight = 256 * minInflight // 4096 max streams per connection on the server side ) - return &dynamicThrottler{ - minInflight: minInflight, - maxInflight: maxInflight, - targetInflight: 8 * minInflight, - targetTimesTen: 10 * maxInflight, + t := &dynamicThrottler{ + minInflight: minInflight, + maxInflight: maxInflight, throughputs: make([]float64, 128), start: nowFunc(), now: nowFunc, } + t.targetInflight.Store(8 * minInflight) + t.targetTimesTen.Store(10 * maxInflight) + return t } func NewThrottler(connections int) Throttler { return newThrottler(connections, time.Now) } func (t *dynamicThrottler) Sent() { - currentInflight := atomic.LoadInt64(&t.targetInflight) + currentInflight := t.targetInflight.Load() t.sent++ if t.sent*t.sent*t.sent < 100*currentInflight*currentInflight { return @@ -64,7 +65,7 @@ func (t *dynamicThrottler) Sent() { now := t.now() elapsed := now.Sub(t.start) t.start = now - currentThroughput := float64(atomic.SwapInt64(&t.ok, 0)) / float64(elapsed) + currentThroughput := float64(t.ok.Swap(0)) / float64(elapsed) // Use buckets for throughput over inflight, along the log-scale, in [minInflight, maxInflight). index := int(float64(len(t.throughputs)) * math.Log(max(1, min(255, float64(currentInflight)/float64(t.minInflight)))) / math.Log(256)) @@ -85,20 +86,20 @@ func (t *dynamicThrottler) Sent() { } } target := int64((rand.Float64()*0.20 + 0.92) * choice) // Random walk, skewed towards increase - atomic.StoreInt64(&t.targetInflight, max(t.minInflight, min(t.maxInflight, target))) + t.targetInflight.Store(max(t.minInflight, min(t.maxInflight, target))) } func (t *dynamicThrottler) Success() { - atomic.AddInt64(&t.targetTimesTen, 1) - atomic.AddInt64(&t.ok, 1) + t.targetTimesTen.Add(1) + t.ok.Add(1) } func (t *dynamicThrottler) Throttled(inflight int64) { - atomic.StoreInt64(&t.targetTimesTen, max(inflight*5, t.minInflight*10)) + t.targetTimesTen.Store(max(inflight*5, t.minInflight*10)) } func (t *dynamicThrottler) TargetInflight() int64 { - staticTargetInflight := min(t.maxInflight, atomic.LoadInt64(&t.targetTimesTen)/10) - targetInflight := atomic.LoadInt64(&t.targetInflight) + staticTargetInflight := min(t.maxInflight, t.targetTimesTen.Load()/10) + targetInflight := t.targetInflight.Load() return min(staticTargetInflight, targetInflight) } diff --git a/client/go/internal/vespa/system.go b/client/go/internal/vespa/system.go index b8263dbdec0..96795cc0ef8 100644 --- a/client/go/internal/vespa/system.go +++ b/client/go/internal/vespa/system.go @@ -4,36 +4,40 @@ import "fmt" // PublicSystem represents the main Vespa Cloud system. var PublicSystem = System{ - Name: "public", - URL: "https://api-ctl.vespa-cloud.com:4443", - ConsoleURL: "https://console.vespa-cloud.com", - DefaultZone: ZoneID{Environment: "dev", Region: "aws-us-east-1c"}, + Name: "public", + URL: "https://api-ctl.vespa-cloud.com:4443", + ConsoleURL: "https://console.vespa-cloud.com", + DefaultZone: ZoneID{Environment: "dev", Region: "aws-us-east-1c"}, + EndpointDomain: "vespa-app.cloud", } // PublicCDSystem represents the CD variant of the Vespa Cloud system. var PublicCDSystem = System{ - Name: "publiccd", - URL: "https://api-ctl.cd.vespa-cloud.com:4443", - ConsoleURL: "https://console.cd.vespa-cloud.com", - DefaultZone: ZoneID{Environment: "dev", Region: "aws-us-east-1c"}, + Name: "publiccd", + URL: "https://api-ctl.cd.vespa-cloud.com:4443", + ConsoleURL: "https://console.cd.vespa-cloud.com", + DefaultZone: ZoneID{Environment: "dev", Region: "aws-us-east-1c"}, + EndpointDomain: "cd.vespa-app.cloud", } // MainSystem represents the main hosted Vespa system. var MainSystem = System{ - Name: "main", - URL: "https://api.vespa.ouryahoo.com:4443", - ConsoleURL: "https://console.vespa.ouryahoo.com", - DefaultZone: ZoneID{Environment: "dev", Region: "us-east-1"}, - AthenzDomain: "vespa.vespa", + Name: "main", + URL: "https://api.vespa.ouryahoo.com:4443", + ConsoleURL: "https://console.vespa.ouryahoo.com", + DefaultZone: ZoneID{Environment: "dev", Region: "us-east-1"}, + AthenzDomain: "vespa.vespa", + EndpointDomain: "vespa.oath.cloud", } // CDSystem represents the CD variant of the hosted Vespa system. var CDSystem = System{ - Name: "cd", - URL: "https://api-cd.vespa.ouryahoo.com:4443", - ConsoleURL: "https://console-cd.vespa.ouryahoo.com", - DefaultZone: ZoneID{Environment: "dev", Region: "cd-us-west-1"}, - AthenzDomain: "vespa.vespa.cd", + Name: "cd", + URL: "https://api-cd.vespa.ouryahoo.com:4443", + ConsoleURL: "https://console-cd.vespa.ouryahoo.com", + DefaultZone: ZoneID{Environment: "dev", Region: "cd-us-west-1"}, + AthenzDomain: "vespa.vespa.cd", + EndpointDomain: "cd.vespa.oath.cloud", } // System represents a Vespa system. @@ -47,6 +51,8 @@ type System struct { // AthenzDomain is the Athenz domain used by this system. This is empty for systems not using Athenz for tenant // authentication. AthenzDomain string + // EndpointDomain is the domain used for application endpoints in this system + EndpointDomain string } // IsPublic returns whether system s is a public (Vespa Cloud) system. diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go index 928bb788494..c0169f1a9bd 100644 --- a/client/go/internal/vespa/target_cloud.go +++ b/client/go/internal/vespa/target_cloud.go @@ -26,6 +26,7 @@ type APIOptions struct { type CloudDeploymentOptions struct { Deployment Deployment TLSOptions TLSOptions + CustomURL string ClusterURLs map[string]string // Endpoints keyed on cluster name } @@ -73,7 +74,15 @@ func CloudTarget(httpClient util.HTTPClient, apiAuth Authenticator, deploymentAu }, nil } -func (t *cloudTarget) findClusterURL(cluster string) (string, error) { +func (t *cloudTarget) findClusterURL(cluster string, timeout time.Duration, runID int64) (string, error) { + if t.deploymentOptions.CustomURL != "" { + return t.deploymentOptions.CustomURL, nil + } + if t.deploymentOptions.ClusterURLs == nil { + if err := t.waitForEndpoints(timeout, runID); err != nil { + return "", err + } + } clusters := make([]string, 0, len(t.deploymentOptions.ClusterURLs)) for c := range t.deploymentOptions.ClusterURLs { clusters = append(clusters, c) @@ -129,12 +138,7 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, c } return service, nil case QueryService, DocumentService: - if t.deploymentOptions.ClusterURLs == nil { - if err := t.waitForEndpoints(timeout, runID); err != nil { - return nil, err - } - } - url, err := t.findClusterURL(cluster) + url, err := t.findClusterURL(cluster, timeout, runID) if err != nil { return nil, err } diff --git a/client/go/internal/vespa/target_custom.go b/client/go/internal/vespa/target_custom.go index 0a3a9d48fed..0129b1e1153 100644 --- a/client/go/internal/vespa/target_custom.go +++ b/client/go/internal/vespa/target_custom.go @@ -41,7 +41,7 @@ func (t *customTarget) Deployment() Deployment { return Deployment{} } func (t *customTarget) createService(name string) (*Service, error) { switch name { case DeployService, QueryService, DocumentService: - url, err := t.urlWithPort(name) + url, err := t.serviceURL(name, t.targetType) if err != nil { return nil, err } @@ -79,20 +79,21 @@ func (t *customTarget) PrintLog(options LogOptions) error { func (t *customTarget) CheckVersion(version version.Version) error { return nil } -func (t *customTarget) urlWithPort(serviceName string) (string, error) { +func (t *customTarget) serviceURL(name string, targetType string) (string, error) { u, err := url.Parse(t.baseURL) if err != nil { return "", err } - port := u.Port() - if port == "" { - switch serviceName { + if targetType == TargetLocal { + // Use same ports as the vespaengine/vespa container image + port := "" + switch name { case DeployService: port = "19071" case QueryService, DocumentService: port = "8080" default: - return "", fmt.Errorf("unknown service: %s", serviceName) + return "", fmt.Errorf("unknown service: %s", name) } u.Host = u.Host + ":" + port } diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index bf266e8f9ec..6dc97f496f5 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -76,9 +76,9 @@ func TestCustomTarget(t *testing.T) { assertServiceURL(t, "http://127.0.0.1:8080", lt, "document") ct := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42", TLSOptions{}) - assertServiceURL(t, "http://192.0.2.42:19071", ct, "deploy") - assertServiceURL(t, "http://192.0.2.42:8080", ct, "query") - assertServiceURL(t, "http://192.0.2.42:8080", ct, "document") + assertServiceURL(t, "http://192.0.2.42", ct, "deploy") + assertServiceURL(t, "http://192.0.2.42", ct, "query") + assertServiceURL(t, "http://192.0.2.42", ct, "document") ct2 := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42:60000", TLSOptions{}) assertServiceURL(t, "http://192.0.2.42:60000", ct2, "deploy") diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 4b993f8e244..ae9d696a9a3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -160,7 +160,7 @@ public class InMemoryProvisioner implements HostProvisioner { public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { provisioned.add(cluster.id(), requested); clusters.add(cluster); - if (environment == Environment.dev) { + if (environment == Environment.dev && ! requested.isRequired()) { requested = requested.withLimits(requested.minResources().withNodes(1), requested.maxResources().withNodes(1)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 4ea74147aaf..475a4174f9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -53,8 +53,8 @@ public class QuotaValidator extends Validator { } throwIfBudgetNegative(actualSpend, budget, systemName); - throwIfBudgetExceeded(actualSpend, budget, systemName); - throwIfBudgetExceeded(maxSpend, budget, systemName); + throwIfBudgetExceeded(actualSpend, budget, systemName, true); + throwIfBudgetExceeded(maxSpend, budget, systemName, false); } private Set<ClusterSpec.Id> adminClusterIds(VespaModel model) { @@ -86,18 +86,22 @@ public class QuotaValidator extends Validator { private static void throwIfBudgetNegative(double spend, BigDecimal budget, SystemName systemName) { if (budget.doubleValue() < 0) { - throw new IllegalArgumentException(quotaMessage("Please free up some capacity.", systemName, spend, budget)); + throw new IllegalArgumentException(quotaMessage("Please free up some capacity.", systemName, spend, budget, true)); } } - private static void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { + private static void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName, boolean actual) { if (budget.doubleValue() < spend) { - throw new IllegalArgumentException(quotaMessage("Contact support to upgrade your plan.", systemName, spend, budget)); + throw new IllegalArgumentException(quotaMessage("Contact support to upgrade your plan.", systemName, spend, budget, actual)); } } - private static String quotaMessage(String message, SystemName system, double spend, BigDecimal budget) { - String quotaDescription = String.format(Locale.ENGLISH, "The max resources specified cost $%.2f but your quota is $%.2f", spend, budget); + private static String quotaMessage(String message, SystemName system, double spend, BigDecimal budget, boolean actual) { + String quotaDescription = String.format(Locale.ENGLISH, + "The %s cost $%.2f but your quota is $%.2f", + actual ? "resources used" : "max resources specified", + spend, + budget); return (system == SystemName.Public ? "" : system.value() + ": ") + quotaDescription + ": " + message; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 1a7b3d62cb7..a1a3b40a858 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -21,19 +21,20 @@ public class QuotaValidatorTest { private final Zone publicZone = new Zone(SystemName.Public, Environment.prod, RegionName.from("foo")); private final Zone publicCdZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("foo")); + private final Zone devZone = new Zone(SystemName.Public, Environment.dev, RegionName.from("foo")); private final Quota quota = Quota.unlimited().withClusterSize(10).withBudget(BigDecimal.valueOf(1.25)); @Test void test_deploy_under_quota() { var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); - tester.deploy(null, getServices("testCluster", 4), Environment.prod, null); + tester.deploy(null, getServices(4), Environment.prod, null); } @Test void test_deploy_above_quota_clustersize() { var tester = new ValidationTester(14, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { - tester.deploy(null, getServices("testCluster", 11), Environment.prod, null); + tester.deploy(null, getServices(11), Environment.prod, null); fail(); } catch (RuntimeException e) { assertEquals("Clusters testCluster exceeded max cluster size of 10", e.getMessage()); @@ -44,10 +45,10 @@ public class QuotaValidatorTest { void test_deploy_above_quota_budget() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { - tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("The max resources specified cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("The resources used cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); } } @@ -55,10 +56,10 @@ public class QuotaValidatorTest { void test_deploy_above_quota_budget_in_publiccd() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { - tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("publiccd: The max resources specified cost $1.63 but your quota is $1.00: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("publiccd: The resources used cost $1.63 but your quota is $1.00: Contact support to upgrade your plan.", e.getMessage()); } } @@ -66,11 +67,33 @@ public class QuotaValidatorTest { void test_deploy_max_resources_above_quota() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); try { - tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("publiccd: The max resources specified cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("publiccd: The resources used cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); + } + } + + + @Test + void test_deploy_above_quota_budget_in_dev() { + var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(0.01)); + var tester = new ValidationTester(5, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(devZone)); + // There is downscaling to 1 node per cluster in dev + try { + tester.deploy(null, getServices(2, false), Environment.dev, null); + fail(); + } catch (RuntimeException e) { + assertEquals("The resources used cost $0.16 but your quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); + } + + // Override so that we will get 2 nodes in content cluster + try { + tester.deploy(null, getServices(2, true), Environment.dev, null); + fail(); + } catch (RuntimeException e) { + assertEquals("The resources used cost $0.33 but your quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); } } @@ -79,25 +102,26 @@ public class QuotaValidatorTest { var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1)); var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { - tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("The max resources specified cost $-.-- but your quota is $--.--: Please free up some capacity.", + assertEquals("The resources used cost $-.-- but your quota is $--.--: Please free up some capacity.", ValidationTester.censorNumbers(e.getMessage())); } } - private static String getServices(String contentClusterId, int nodeCount) { - return "<services version='1.0'>" + - " <content id='" + contentClusterId + "' version='1.0'>" + + private static String getServices(int nodeCount) { + return getServices(nodeCount, false); + } + + private static String getServices(int nodeCount, boolean devOverride) { + return "<services version='1.0' xmlns:deploy='vespa' xmlns:preprocess='properties'>" + + " <content id='" + "testCluster" + "' version='1.0'>" + " <redundancy>1</redundancy>" + - " <engine>" + - " <proton/>" + - " </engine>" + " <documents>" + " <document type='music' mode='index'/>" + " </documents>" + - " <nodes count='" + nodeCount + "'>" + + " <nodes count='" + nodeCount + "' " + (devOverride ? "required='true'" : "") + " >\n" + " <resources vcpu=\"[0.5, 2]\" memory=\"[1Gb, 6Gb]\" disk=\"[1Gb, 18Gb]\"/>\n" + " </nodes>" + " </content>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 78d3838d39d..1517f7971ed 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -57,6 +57,7 @@ public class ValidationTester { public ValidationTester(InMemoryProvisioner hostProvisioner, TestProperties testProperties) { this.hostProvisioner = hostProvisioner; this.properties = testProperties; + hostProvisioner.setEnvironment(testProperties.zone().environment()); } /** diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 29c5fa69429..f2334a6ef00 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -15,7 +15,7 @@ commons-io:commons-io:2.11.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 org.apache.commons:commons-collections4:4.2 -org.apache.commons:commons-compress:1.22 +org.apache.commons:commons-compress:1.23.0 org.apache.commons:commons-lang3:3.12.0 org.apache.maven:maven-archiver:3.6.0 org.apache.maven:maven-artifact:3.8.7 diff --git a/parent/pom.xml b/parent/pom.xml index 71e0c35eb6c..f6fdaf20a7a 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -716,7 +716,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> - <version>1.22</version> + <version>1.23.0</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 7c2db779693..de2ee3fb412 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -90,7 +90,7 @@ net.openhft:zero-allocation-hashing:0.16 org.antlr:antlr-runtime:3.5.3 org.antlr:antlr4-runtime:4.11.1 org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.6 -org.apache.commons:commons-compress:1.22 +org.apache.commons:commons-compress:1.23.0 org.apache.commons:commons-csv:1.8 org.apache.commons:commons-exec:1.3 org.apache.commons:commons-lang3:3.12.0 |