diff options
36 files changed, 270 insertions, 237 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java index 5138bee1ff6..5143a38b2c1 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java @@ -3,10 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.net.HostName; -import com.yahoo.security.KeyUtils; - import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; @@ -35,20 +32,17 @@ public class IdentityDocumentGenerator { private final NodeRepository nodeRepository; private final Zone zone; private final KeyProvider keyProvider; - private final SecretStore secretStore; private final AthenzProviderServiceConfig athenzProviderServiceConfig; @Inject public IdentityDocumentGenerator(AthenzProviderServiceConfig config, NodeRepository nodeRepository, Zone zone, - KeyProvider keyProvider, - SecretStore secretStore) { + KeyProvider keyProvider) { this.athenzProviderServiceConfig = config; this.nodeRepository = nodeRepository; this.zone = zone; this.keyProvider = keyProvider; - this.secretStore = secretStore; } public SignedIdentityDocument generateSignedIdentityDocument(String hostname, IdentityType identityType) { @@ -67,7 +61,7 @@ public class IdentityDocumentGenerator { Set<String> ips = new HashSet<>(node.ipConfig().primary()); - PrivateKey privateKey = privateKey(node); + PrivateKey privateKey = keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); AthenzService providerService = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); String configServerHostname = HostName.getLocalhost(); @@ -79,28 +73,11 @@ public class IdentityDocumentGenerator { return new SignedIdentityDocument( signature, athenzProviderServiceConfig.secretVersion(), providerUniqueId, providerService, SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION, configServerHostname, node.hostname(), - createdAt, ips, identityType, clusterType, ztsUrl(node)); + createdAt, ips, identityType, clusterType); } catch (Exception e) { throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e); } } - private PrivateKey privateKey(Node node) { - // return sisSecret for public non-enclave hosts. secret otherwise - if (zone.system().isPublic() && !node.cloudAccount().isEnclave(zone)) { - String keyPem = secretStore.getSecret(athenzProviderServiceConfig.sisSecretName(), athenzProviderServiceConfig.sisSecretVersion()); - return KeyUtils.fromPemEncodedPrivateKey(keyPem); - } else { - return keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); - } - } - private String ztsUrl(Node node) { - // return sisUrl for public non-enclave hosts, ztsUrl otherwise - if (zone.system().isPublic() && !node.cloudAccount().isEnclave(zone)) { - return athenzProviderServiceConfig.sisUrl(); - } else { - return athenzProviderServiceConfig.ztsUrl(); - } - } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 231f22ac56b..531a815922b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -65,7 +65,7 @@ public class CertificateAuthorityApiHandler extends ThreadedHttpRequestHandler { super(ctx); this.secretStore = secretStore; this.certificates = certificates; - this.caPrivateKeySecretName = athenzProviderServiceConfig.sisSecretName(); + this.caPrivateKeySecretName = athenzProviderServiceConfig.secretName(); this.caCertificateSecretName = athenzProviderServiceConfig.caCertSecretName(); this.instanceValidator = instanceValidator; } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java index 8c575a6403b..fec03afab69 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java @@ -49,7 +49,6 @@ public class InstanceSerializer { private static final String IDD_IPADDRESSES_FIELD = "ip-addresses"; private static final String IDD_IDENTITY_TYPE_FIELD = "identity-type"; private static final String IDD_CLUSTER_TYPE_FIELD = "cluster-type"; - private static final String IDD_ZTS_URL_FIELD = "zts-url"; private static final ObjectMapper objectMapper = new ObjectMapper(); static { @@ -101,12 +100,10 @@ public class InstanceSerializer { IdentityType identityType = IdentityType.fromId(requireField(IDD_IDENTITY_TYPE_FIELD, root).asString()); var clusterTypeField = root.field(IDD_CLUSTER_TYPE_FIELD); var clusterType = clusterTypeField.valid() ? ClusterType.from(clusterTypeField.asString()) : null; - var ztsUrlField = root.field(IDD_ZTS_URL_FIELD); - var ztsUrl = ztsUrlField.valid() ? ztsUrlField.asString() : ""; return new SignedIdentityDocument(signature, (int)signingKeyVersion, providerUniqueId, athenzService, (int)documentVersion, - configserverHostname, instanceHostname, createdAt, ips, identityType, clusterType, ztsUrl); + configserverHostname, instanceHostname, createdAt, ips, identityType, clusterType); } private static Instant getJsr310Instant(double v) { diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java index 340be33c2a3..9205baff0fc 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -75,12 +74,11 @@ public class IdentityDocumentGeneratorTest { when(nodes.node(eq(parentHostname))).thenReturn(Optional.of(parentNode)); when(nodes.node(eq(containerHostname))).thenReturn(Optional.of(containerNode)); AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - SecretStoreMock secretStore = new SecretStoreMock(); String dnsSuffix = "vespa.dns.suffix"; AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix); IdentityDocumentGenerator identityDocumentGenerator = - new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider, secretStore); + new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); // Verify attributes diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java index e7355c75d8e..a7947aff283 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java @@ -225,7 +225,7 @@ public class InstanceValidatorTest { IdentityType.NODE, keyProvider.getPrivateKey(0)); SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( signature, 0, vespaUniqueInstanceId, domainService, 0, "localhost", "localhost", - clock, Collections.emptySet(), IdentityType.NODE, clusterType, "https://zts.url"); + clock, Collections.emptySet(), IdentityType.NODE, clusterType); return createInstanceConfirmation(vespaUniqueInstanceId, domain, service, signedIdentityDocument); } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java index d880fd5220b..8112f5779e5 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java @@ -64,8 +64,6 @@ public class ContainerTester { " <serviceName>servicename</serviceName>\n" + " <secretName>secretname</secretName>\n" + " <secretVersion>0</secretVersion>\n" + - " <sisSecretName>secretname</sisSecretName>\n" + - " <sisSecretVersion>0</sisSecretVersion>\n" + " <caCertSecretName>vespa.external.ca.cert</caCertSecretName>\n" + " <certDnsSuffix>suffix</certDnsSuffix>\n" + " <ztsUrl>https://localhost:123/</ztsUrl>\n" + diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java index 02398b19627..ca624918beb 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.ca.restapi; -import com.yahoo.config.provision.Cloud; import com.yahoo.security.Pkcs10CsrUtils; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; @@ -49,8 +48,7 @@ public class InstanceSerializerTest { Instant.now().truncatedTo(ChronoUnit.MICROS), // Truncate to the precision given from EntityBindingsMapper.toAttestationData() Collections.emptySet(), IdentityType.NODE, - ClusterType.CONTAINER, - "https://zts.url"); + ClusterType.CONTAINER); var json = String.format("{\n" + " \"provider\": \"provider_prod_us-north-1\",\n" + diff --git a/client/go/go.mod b/client/go/go.mod index ad099ead3bb..d03f22a9e67 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -13,7 +13,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/zalando/go-keyring v0.1.1 - golang.org/x/sys v0.1.0 + golang.org/x/sys v0.5.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/client/go/go.sum b/client/go/go.sum index 9fd929d8412..084bde701ce 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -4,13 +4,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.1.0 h1:3RNcEpBg4IhIChZdFRSdlQt1QjCp1sMAPIrOnm7Yf8g= github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -22,11 +19,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 h1:acNfDZXmm28D2Yg/c3ALnZStzNaZMSagpbr96vY6Zjc= @@ -39,26 +33,17 @@ github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/go/internal/vespa/detect_hostname.go b/client/go/internal/vespa/detect_hostname.go index 9d32ae1a2fd..9729fd80400 100644 --- a/client/go/internal/vespa/detect_hostname.go +++ b/client/go/internal/vespa/detect_hostname.go @@ -4,10 +4,14 @@ package vespa import ( + "fmt" "net" "os" + "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" + "github.com/vespa-engine/vespa/client/go/internal/util" ) // detect if this host is IPv6-only, in which case we want to pass @@ -52,8 +56,76 @@ func FindOurHostname() (string, error) { if err != nil { name = "localhost" } + name, err = findOurHostnameFrom(name) if err == nil { os.Setenv(envvars.VESPA_HOSTNAME, name) } return name, err } + +func validateHostname(name string) bool { + ipAddrs, _ := net.LookupIP(name) + trace.Debug("lookupIP", name, "=>", ipAddrs) + if len(ipAddrs) < 1 { + return false + } + myIpAddresses := make(map[string]bool) + interfaceAddrs, _ := net.InterfaceAddrs() + for _, ifAddr := range interfaceAddrs { + // note: ifAddr.String() is typically "127.0.0.1/8" + if ipnet, ok := ifAddr.(*net.IPNet); ok { + myIpAddresses[ipnet.IP.String()] = true + } + } + trace.Debug("validate with interfaces =>", myIpAddresses) + someGood := false + for _, addr := range ipAddrs { + if len(myIpAddresses) == 0 { + // no validation possible, assume OK + return true + } + if myIpAddresses[addr.String()] { + someGood = true + } else { + return false + } + } + return someGood +} + +func goodHostname(name string) (result string, good bool) { + result = strings.TrimSpace(name) + result = strings.TrimSuffix(result, ".") + if name != result { + trace.Trace("trimmed hostname", name, "=>", result) + } + good = strings.Contains(result, ".") && validateHostname(result) + trace.Debug("hostname:", result, "good =>", good) + return +} + +func findOurHostnameFrom(name string) (string, error) { + trimmed, good := goodHostname(name) + if good { + return trimmed, nil + } + backticks := util.BackTicksIgnoreStderr + out, err := backticks.Run("vespa-detect-hostname") + if err != nil { + out, err = backticks.Run("hostname", "-f") + } + if err != nil { + out, err = backticks.Run("hostname") + } + alternate, good := goodHostname(out) + if err == nil && good { + return alternate, nil + } + if validateHostname(trimmed) { + return trimmed, nil + } + if validateHostname(alternate) { + return alternate, nil + } + return "localhost", fmt.Errorf("fallback to localhost [os.Hostname was '%s']", name) +} diff --git a/client/go/internal/vespa/detect_hostname_test.go b/client/go/internal/vespa/detect_hostname_test.go index 4afb139743d..324701efb6f 100644 --- a/client/go/internal/vespa/detect_hostname_test.go +++ b/client/go/internal/vespa/detect_hostname_test.go @@ -4,6 +4,7 @@ package vespa import ( "fmt" "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -17,7 +18,16 @@ func TestDetectHostname(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "foo.bar", got) os.Unsetenv("VESPA_HOSTNAME") - got, _ = FindOurHostname() + got, err = findOurHostnameFrom("bar.foo.123") + fmt.Fprintln(os.Stderr, "findOurHostname from bar.foo.123 returns:", got, "with error:", err) assert.NotEqual(t, "", got) - fmt.Fprintln(os.Stderr, "FindOurHostname() returns:", got, "with error:", err) + parts := strings.Split(got, ".") + if len(parts) > 1 { + expanded, err2 := findOurHostnameFrom(parts[0]) + fmt.Fprintln(os.Stderr, "findOurHostname from", parts[0], "returns:", expanded, "with error:", err2) + assert.Equal(t, got, expanded) + } + got, err = findOurHostnameFrom("") + assert.NotEqual(t, "", got) + fmt.Fprintln(os.Stderr, "findOurHostname('') returns:", got, "with error:", err) } diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 4c9c74f9b8f..2131aa88d30 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -13,11 +13,6 @@ secretName string # Secret version secretVersion int -# Tempory resources -sisSecretName string default="" -sisSecretVersion int default=0 -sisUrl string default = "" - # Secret name of CA certificate caCertSecretName string diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 898e348db92..2d6e059342e 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -10,6 +10,8 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.yql.MinimalQueryInserter; +import com.yahoo.yolean.chain.After; /** * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. @@ -17,6 +19,7 @@ import com.yahoo.search.searchchain.Execution; * * @author karowan */ +@After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); static final CompoundName WAND_HITS = new CompoundName("wand.hits"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 43aaba7b0f9..33f840c7af0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -44,11 +44,9 @@ import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.query.Sorting.UcaSorter; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; import java.util.ArrayList; import java.util.Collection; @@ -68,7 +66,6 @@ public class YqlParserTestCase { private final YqlParser parser = new YqlParser(new ParserEnvironment()); @Test - @Timeout(120_000) void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10", new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 token recognition error at: 'œ'")); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java new file mode 100644 index 00000000000..e6dec99b84c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Optional; + +/** + * Represents an operation to update or unset the archive URI value for a given tenant or cloud account. + * + * @author freva + */ +public class ArchiveUriUpdate { + private final Optional<TenantName> tenantName; + private final Optional<CloudAccount> cloudAccount; + private final Optional<URI> archiveUri; + + private ArchiveUriUpdate(Optional<TenantName> tenantName, Optional<CloudAccount> cloudAccount, Optional<URI> archiveUri) { + this.tenantName = tenantName; + this.cloudAccount = cloudAccount; + this.archiveUri = archiveUri; + } + + public Optional<TenantName> tenantName() { return tenantName; } + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + public Optional<URI> archiveUri() { return archiveUri; } + + public static ArchiveUriUpdate setArchiveUriFor(TenantName tenantName, URI archiveUri) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(TenantName tenantName) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.empty()); + } + + public static ArchiveUriUpdate setArchiveUriFor(CloudAccount cloudAccount, URI archiveUri) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(CloudAccount cloudAccount) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.empty()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java new file mode 100644 index 00000000000..a0f6955b59f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Map; + +/** + * @author freva + */ +public record ArchiveUris(Map<TenantName, URI> tenantArchiveUris, Map<CloudAccount, URI> accountArchiveUris) { + public static final ArchiveUris EMPTY = new ArchiveUris(Map.of(), Map.of()); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 1768de8d012..4c5a67626ea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -5,15 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; -import javax.ws.rs.HeaderParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import java.net.URI; import java.util.List; import java.util.Map; @@ -48,14 +43,11 @@ public interface NodeRepository { /** Get node statistics such as cost and load from given zone */ NodeRepoStats getStats(ZoneId zone); - /** Get all archive URLs found in zone */ - Map<TenantName, URI> getArchiveUris(ZoneId zone); + /** Get all archive URIs found in zone */ + ArchiveUris getArchiveUris(ZoneId zone); - /** Update archive URL for given tenant */ - void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri); - - /** Remove archive URL for given tenant */ - void removeArchiveUri(ZoneId zone, TenantName tenantName); + /** Update some archive URI in the given zone */ + void updateArchiveUri(ZoneId zone, ArchiveUriUpdate archiveUriUpdate); /** Upgrade all nodes of given type to a new version */ void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java index 274d07bfc3b..172523eb261 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java @@ -16,6 +16,9 @@ public class ArchiveList { @JsonProperty("tenant") public String tenant; + @JsonProperty("account") + public String account; + @JsonProperty("uri") public String uri; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 0c8a50fa821..ddb1365f2de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -5,12 +5,13 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.yolean.Exceptions; -import java.net.URI; import java.time.Duration; import java.util.HashMap; import java.util.HashSet; @@ -56,17 +57,19 @@ public class ArchiveUriUpdater extends ControllerMaintainer { int failures = 0; for (ZoneId zone : tenantsByZone.keySet()) { try { - Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone); + ArchiveUris zoneArchiveUris = nodeRepository.getArchiveUris(zone); for (TenantName tenant : tenantsByZone.get(zone)) { archiveBucketDb.archiveUriFor(zone, tenant, true) - .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant))) - .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri)); + .filter(uri -> !uri.equals(zoneArchiveUris.tenantArchiveUris().get(tenant))) + .ifPresent(uri -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, uri))); } - zoneArchiveUris.keySet().stream() + zoneArchiveUris.tenantArchiveUris().keySet().stream() .filter(tenant -> !tenantsByZone.get(zone).contains(tenant)) - .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant)); + .forEach(tenant -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.deleteArchiveUriFor(tenant))); + + // TODO (freva): Update account archive URIs } catch (Exception e) { log.log(Level.WARNING, "Failed to update archive URI in " + zone + ". Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 37ef85b991b..297997365b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -11,8 +12,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationStats; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; @@ -22,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVers import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -41,7 +45,7 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<ZoneId, TargetVersions> targetVersions = new ConcurrentHashMap<>(); private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new ConcurrentHashMap<>(); private final Map<DeploymentClusterId, BcpGroupInfo> bcpGroupInfos = new ConcurrentHashMap<>(); - private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new ConcurrentHashMap<>(); + private final Map<ZoneId, ArchiveUris> archiveUris = new ConcurrentHashMap<>(); private boolean allowPatching = true; private boolean hasSpareCapacity = false; @@ -118,18 +122,26 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public Map<TenantName, URI> getArchiveUris(ZoneId zone) { - return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); + public ArchiveUris getArchiveUris(ZoneId zone) { + return archiveUris.getOrDefault(zone, ArchiveUris.EMPTY); } @Override - public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) { - archiveUris.computeIfAbsent(zone, z -> new ConcurrentHashMap<>()).put(tenantName, archiveUri); - } - - @Override - public void removeArchiveUri(ZoneId zone, TenantName tenantName) { - Optional.ofNullable(archiveUris.get(zone)).ifPresent(map -> map.remove(tenantName)); + public void updateArchiveUri(ZoneId zone, ArchiveUriUpdate update) { + archiveUris.compute(zone, (z, prev) -> { + prev = prev == null ? ArchiveUris.EMPTY : prev; + if (update.tenantName().isPresent()) { + Map<TenantName, URI> updated = new HashMap<>(prev.tenantArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.tenantName().get(), uri), + () -> updated.remove(update.tenantName().get())); + return new ArchiveUris(updated, prev.accountArchiveUris()); + } else { + Map<CloudAccount, URI> updated = new HashMap<>(prev.accountArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.cloudAccount().get(), uri), + () -> updated.remove(update.cloudAccount().get())); + return new ArchiveUris(prev.tenantArchiveUris(), updated); + } + }); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 8c44b39691c..de62a2fc48c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -62,7 +62,7 @@ public class ArchiveUriUpdaterTest { private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { Map<TenantName, String> actualUris = tester.controller().serviceRegistry().configServer().nodeRepository() - .getArchiveUris(zone).entrySet().stream() + .getArchiveUris(zone).tenantArchiveUris().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); assertEquals(expectedUris, actualUris); } @@ -76,7 +76,7 @@ public class ArchiveUriUpdaterTest { private void setArchiveUriInNodeRepo(Map<TenantName, String> archiveUris, ZoneId zone) { NodeRepository nodeRepository = tester.controller().serviceRegistry().configServer().nodeRepository(); - archiveUris.forEach((tenant, uri) -> nodeRepository.setArchiveUri(zone, tenant, URI.create(uri))); + archiveUris.forEach((tenant, uri) -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, URI.create(uri)))); } private void deploy(DeploymentContext application, ZoneId zone) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9b4b04a3d62..3989b45b9ac 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -345,13 +345,6 @@ public class Flags { "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); - public static final UnboundBooleanFlag VESPA_ATHENZ_PROVIDER = defineFeatureFlag( - "vespa-athenz-provider", false, - List.of("mortent"), "2023-02-22", "2023-05-01", - "Enable athenz provider in public systems", - "Takes effect on next config server container start", - ZONE_ID); - /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/messagebus/src/vespa/messagebus/messenger.cpp b/messagebus/src/vespa/messagebus/messenger.cpp index 20b39dae522..926d8b6ef22 100644 --- a/messagebus/src/vespa/messagebus/messenger.cpp +++ b/messagebus/src/vespa/messagebus/messenger.cpp @@ -18,70 +18,6 @@ struct DeleteFunctor } }; -class MessageTask : public mbus::Messenger::ITask { -private: - mbus::Message::UP _msg; - mbus::IMessageHandler &_handler; - -public: - MessageTask(mbus::Message::UP msg, mbus::IMessageHandler &handler) - : _msg(std::move(msg)), - _handler(handler) - { - // empty - } - - ~MessageTask() { - if (_msg) { - _msg->discard(); - } - } - - void run() override { - _handler.handleMessage(std::move(_msg)); - } - - uint8_t priority() const override { - if (_msg) { - return _msg->priority(); - } - - return 255; - } -}; - -class ReplyTask : public mbus::Messenger::ITask { -private: - mbus::Reply::UP _reply; - mbus::IReplyHandler &_handler; - -public: - ReplyTask(mbus::Reply::UP reply, mbus::IReplyHandler &handler) - : _reply(std::move(reply)), - _handler(handler) - { - // empty - } - - ~ReplyTask() { - if (_reply) { - _reply->discard(); - } - } - - void run() override { - _handler.handleReply(std::move(_reply)); - } - - uint8_t priority() const override { - if (_reply) { - return _reply->priority(); - } - - return 255; - } -}; - class SyncTask : public mbus::Messenger::ITask { private: vespalib::Gate &_gate; @@ -89,17 +25,13 @@ private: public: SyncTask(vespalib::Gate &gate) : _gate(gate) - { - // empty - } + { } ~SyncTask() { _gate.countDown(); } - void run() override { - // empty - } + void run() override { } uint8_t priority() const override { return 255; @@ -115,9 +47,7 @@ public: AddRecurrentTask(std::vector<ITask*> &tasks, mbus::Messenger::ITask::UP task) : _tasks(tasks), _task(std::move(task)) - { - // empty - } + { } void run() override { _tasks.push_back(_task.release()); @@ -136,9 +66,7 @@ public: DiscardRecurrentTasks(vespalib::Gate &gate, std::vector<ITask*> &tasks) : SyncTask(gate), _tasks(tasks) - { - // empty - } + { } void run() override { std::for_each(_tasks.begin(), _tasks.end(), DeleteFunctor<ITask>()); diff --git a/messagebus/src/vespa/messagebus/replygate.cpp b/messagebus/src/vespa/messagebus/replygate.cpp index d1bd6ef05c7..8094119f14c 100644 --- a/messagebus/src/vespa/messagebus/replygate.cpp +++ b/messagebus/src/vespa/messagebus/replygate.cpp @@ -38,9 +38,8 @@ ReplyGate::handleReply(Reply::UP reply) } void -ReplyGate::handleDiscard(Context ctx) +ReplyGate::handleDiscard(Context) { - (void)ctx; subRef(); } diff --git a/messagebus/src/vespa/messagebus/sendproxy.cpp b/messagebus/src/vespa/messagebus/sendproxy.cpp index dcb24b70eeb..9adeceb9a41 100644 --- a/messagebus/src/vespa/messagebus/sendproxy.cpp +++ b/messagebus/src/vespa/messagebus/sendproxy.cpp @@ -13,9 +13,7 @@ SendProxy::SendProxy(MessageBus &mbus, INetwork &net, Resender *resender) : _msg(), _logTrace(false), _root() -{ - // empty -} +{ } void SendProxy::handleMessage(Message::UP msg) @@ -36,9 +34,8 @@ SendProxy::handleMessage(Message::UP msg) } void -SendProxy::handleDiscard(Context ctx) +SendProxy::handleDiscard(Context) { - (void)ctx; _msg->discard(); delete this; } diff --git a/messagebus/src/vespa/messagebus/sequencer.cpp b/messagebus/src/vespa/messagebus/sequencer.cpp index e17509033d6..59e3164e11d 100644 --- a/messagebus/src/vespa/messagebus/sequencer.cpp +++ b/messagebus/src/vespa/messagebus/sequencer.cpp @@ -18,8 +18,8 @@ Sequencer::Sequencer(IMessageHandler &sender) : Sequencer::~Sequencer() { - for (QueueMap::iterator it = _seqMap.begin(); it != _seqMap.end(); ++it) { - MessageQueue *queue = it->second; + for (auto & entry : _seqMap) { + MessageQueue *queue = entry.second; if (queue != nullptr) { while (queue->size() > 0) { Message *msg = queue->front(); @@ -39,7 +39,7 @@ Sequencer::filter(Message::UP msg) msg->setContext(Context(seqId)); { std::lock_guard guard(_lock); - QueueMap::iterator it = _seqMap.find(seqId); + auto it = _seqMap.find(seqId); if (it != _seqMap.end()) { if (it->second == nullptr) { it->second = new MessageQueue(); @@ -48,7 +48,7 @@ Sequencer::filter(Message::UP msg) make_string("Sequencer queued message with sequence id '%" PRIu64 "'.", seqId)); it->second->push(msg.get()); msg.release(); - return Message::UP(); + return {}; } _seqMap[seqId] = nullptr; // insert empty queue } @@ -87,7 +87,7 @@ Sequencer::handleReply(Reply::UP reply) Message::UP msg; { std::lock_guard guard(_lock); - QueueMap::iterator it = _seqMap.find(seq); + auto it = _seqMap.find(seq); MessageQueue *que = it->second; assert(it != _seqMap.end()); if (que == nullptr || que->size() == 0) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 6bd7d98e207..fc49dcc744c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -41,7 +41,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; @@ -190,9 +189,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - // Allow all zts hosts while removing SIS - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis + HostnameVerifier ztsHostNameVerifier = useInternalZts + ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) + : null; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.registerInstance( configserverIdentity, @@ -205,15 +206,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } - /** - * Return zts url from identity document, fallback to ztsEndpoint - */ - private URI ztsEndpoint(SignedIdentityDocument doc) { - return Optional.ofNullable(doc.ztsUrl()) - .filter(s -> !s.isBlank()) - .map(URI::create) - .orElse(ztsEndpoint); - } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, SignedIdentityDocument doc) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); @@ -225,9 +217,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .build(); try { - // Allow all zts hosts while removing SIS - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis + HostnameVerifier ztsHostNameVerifier = useInternalZts + ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) + : null; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.refreshInstance( configserverIdentity, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 4020166a132..f52c4cc85f7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -62,7 +62,7 @@ public class Autoscaler { clusterNodes, nodeRepository.metricsDb(), nodeRepository.clock()); - if (clusterModel.isEmpty()) return Autoscaling.empty(); + if (clusterModel.isEmpty()) return Autoscaling.empty(clusterModel.description()); if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources())) return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", clusterModel); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 3825309e97b..9506bba73e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -102,8 +102,12 @@ public class Autoscaling { } public static Autoscaling empty() { + return empty(""); + } + + public static Autoscaling empty(String description) { return new Autoscaling(Status.unavailable, - "", + description, Optional.empty(), Instant.EPOCH, Load.zero(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 4f262fb8105..3f916f1cc59 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -113,6 +113,10 @@ public class ClusterModel { public ClusterSpec clusterSpec() { return clusterSpec; } public Cluster cluster() { return cluster; } + public String description() { + return nodeTimeseries.description(); + } + public boolean isEmpty() { return nodeTimeseries().isEmpty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index 6c02f7466eb..10f3b54dd45 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -23,6 +23,8 @@ public class ClusterNodesTimeseries { /** The measurements for all nodes in this snapshot */ private final List<NodeTimeseries> timeseries; + private int initialCount, afterWarmupCount, afterStableCount, finalCount; + public ClusterNodesTimeseries(Duration period, Cluster cluster, NodeList clusterNodes, MetricsDb db) { this.clusterNodes = clusterNodes; @@ -31,12 +33,16 @@ public class ClusterNodesTimeseries { // If either this is the case, or there is a generation change, we ignore // the first warmupWindow metrics. var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(4)), clusterNodes); + initialCount = totalMeasurementsIn(timeseries); if (cluster.lastScalingEvent().isPresent()) { long currentGeneration = cluster.lastScalingEvent().get().generation(); timeseries = keepGenerationAfterWarmup(timeseries, currentGeneration); + afterWarmupCount = totalMeasurementsIn(timeseries); } timeseries = keep(timeseries, snapshot -> snapshot.inService() && snapshot.stable()); + afterStableCount = totalMeasurementsIn(timeseries); timeseries = keep(timeseries, snapshot -> ! snapshot.at().isBefore(db.clock().instant().minus(period))); + finalCount = totalMeasurementsIn(timeseries); this.timeseries = timeseries; } @@ -45,6 +51,13 @@ public class ClusterNodesTimeseries { this.timeseries = timeseries; } + public String description() { + return "initial measurements: " + initialCount + + ", after warmpup: " + afterWarmupCount + + ", after stable: " + afterStableCount + + ", final measurements: " + finalCount; + } + public boolean isEmpty() { return measurementsPerNode() == 0; } @@ -52,8 +65,11 @@ public class ClusterNodesTimeseries { /** Returns the average number of measurements per node */ public double measurementsPerNode() { if (clusterNodes.size() == 0) return 0; - int measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); - return (double)measurementCount / clusterNodes.size(); + return (double) totalMeasurementsIn(timeseries) / clusterNodes.size(); + } + + private int totalMeasurementsIn(List<NodeTimeseries> timeseries) { + return timeseries.stream().mapToInt(m -> m.size()).sum(); } /** Returns the number of nodes measured in this */ diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 067e8a6b00f..9b7b666e353 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -58,7 +58,6 @@ public class EntityBindingsMapper { entity.ipAddresses(), IdentityType.fromId(entity.identityType()), Optional.ofNullable(entity.clusterType()).map(ClusterType::from).orElse(null), - entity.ztsUrl(), entity.unknownAttributes()); } @@ -75,7 +74,6 @@ public class EntityBindingsMapper { model.ipAddresses(), model.identityType().id(), Optional.ofNullable(model.clusterType()).map(ClusterType::toConfigValue).orElse(null), - model.ztsUrl(), model.unknownAttributes()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 6661ac47b81..49a39d25e87 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.yahoo.vespa.athenz.api.AthenzService; -import java.net.URL; import java.time.Instant; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -17,19 +17,25 @@ import java.util.Set; public record SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl, Map<String, Object> unknownAttributes) { + IdentityType identityType, ClusterType clusterType, Map<String, Object> unknownAttributes) { public SignedIdentityDocument { ipAddresses = Set.copyOf(ipAddresses); - unknownAttributes = Map.copyOf(unknownAttributes); + + Map<String, Object> nonNull = new HashMap<>(); + unknownAttributes.forEach((key, value) -> { + if (value != null) nonNull.put(key, value); + }); + // Map.copyOf() does not allow null values + unknownAttributes = Map.copyOf(nonNull); } public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl) { + IdentityType identityType, ClusterType clusterType) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, Map.of()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, Map.of()); } public static final int DEFAULT_DOCUMENT_VERSION = 2; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index edbe032ec26..c37dd2f9147 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -17,7 +17,7 @@ import java.util.Set; public record SignedIdentityDocumentEntity( String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, String ztsUrl, Map<String, Object> unknownAttributes) { + String identityType, String clusterType, Map<String, Object> unknownAttributes) { @JsonCreator public SignedIdentityDocumentEntity(@JsonProperty("signature") String signature, @@ -30,10 +30,9 @@ public record SignedIdentityDocumentEntity( @JsonProperty("created-at") Instant createdAt, @JsonProperty("ip-addresses") Set<String> ipAddresses, @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType, - @JsonProperty("zts-url") String ztsUrl) { + @JsonProperty("cluster-type") String clusterType) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, new HashMap<>()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, new HashMap<>()); } @JsonProperty("signature") @Override public String signature() { return signature; } @@ -47,7 +46,6 @@ public record SignedIdentityDocumentEntity( @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } @JsonProperty("identity-type") @Override public String identityType() { return identityType; } @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } - @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index 2a68f6fd231..f8c119190a6 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -30,7 +30,6 @@ class EntityBindingsMapperTest { "ip-addresses": [], "identity-type": "node", "cluster-type": "admin", - "zts-url": "https://zts.url/", "unknown-string": "string-value", "unknown-object": { "member-in-unknown-object": 123 } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 72798b03fa8..0b8ff4277f1 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -36,7 +36,6 @@ public class IdentityDocumentSignerTest { private static final Instant createdAt = Instant.EPOCH; private static final HashSet<String> ipAddresses = new HashSet<>(Arrays.asList("1.2.3.4", "::1")); private static final ClusterType clusterType = ClusterType.CONTAINER; - private static final String ztsUrl = "https://foo"; @Test void generates_and_validates_signature() { @@ -47,27 +46,27 @@ public class IdentityDocumentSignerTest { SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl); + instanceHostname, createdAt, ipAddresses, identityType, clusterType); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } @Test - void ignores_cluster_type_and_zts_url() { + void ignores_cluster_type() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); String signature = signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, ipAddresses, identityType, keyPair.getPrivate()); - var docWithoutIgnoredFields = new SignedIdentityDocument( + var docWithoutClusterType = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null, null); - var docWithIgnoredFields = new SignedIdentityDocument( + instanceHostname, createdAt, ipAddresses, identityType, null); + var docWithClusterType = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl); + instanceHostname, createdAt, ipAddresses, identityType, clusterType); - assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); - assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); + assertTrue(signer.hasValidSignature(docWithoutClusterType, keyPair.getPublic())); + assertEquals(docWithClusterType.signature(), docWithoutClusterType.signature()); } }
\ No newline at end of file |