diff options
139 files changed, 577 insertions, 827 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 5463c4215a0..ce11196725f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,8 @@ vespa_use_default_cmake_prefix_path() SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) +find_package(Threads REQUIRED) + find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -99,7 +101,6 @@ add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) -add_subdirectory(fastos) add_subdirectory(fbench) add_subdirectory(fileacquirer) add_subdirectory(filedistribution) diff --git a/README-cmake.md b/README-cmake.md index 214eaaa94a9..393e068d089 100644 --- a/README-cmake.md +++ b/README-cmake.md @@ -106,7 +106,6 @@ The module definition is used to specify common dependencies for every target de vespa_define_module( DEPENDS - fastos vespalib bjarnelib diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java index 5138bee1ff6..5143a38b2c1 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java @@ -3,10 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.net.HostName; -import com.yahoo.security.KeyUtils; - import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; @@ -35,20 +32,17 @@ public class IdentityDocumentGenerator { private final NodeRepository nodeRepository; private final Zone zone; private final KeyProvider keyProvider; - private final SecretStore secretStore; private final AthenzProviderServiceConfig athenzProviderServiceConfig; @Inject public IdentityDocumentGenerator(AthenzProviderServiceConfig config, NodeRepository nodeRepository, Zone zone, - KeyProvider keyProvider, - SecretStore secretStore) { + KeyProvider keyProvider) { this.athenzProviderServiceConfig = config; this.nodeRepository = nodeRepository; this.zone = zone; this.keyProvider = keyProvider; - this.secretStore = secretStore; } public SignedIdentityDocument generateSignedIdentityDocument(String hostname, IdentityType identityType) { @@ -67,7 +61,7 @@ public class IdentityDocumentGenerator { Set<String> ips = new HashSet<>(node.ipConfig().primary()); - PrivateKey privateKey = privateKey(node); + PrivateKey privateKey = keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); AthenzService providerService = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); String configServerHostname = HostName.getLocalhost(); @@ -79,28 +73,11 @@ public class IdentityDocumentGenerator { return new SignedIdentityDocument( signature, athenzProviderServiceConfig.secretVersion(), providerUniqueId, providerService, SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION, configServerHostname, node.hostname(), - createdAt, ips, identityType, clusterType, ztsUrl(node)); + createdAt, ips, identityType, clusterType); } catch (Exception e) { throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e); } } - private PrivateKey privateKey(Node node) { - // return sisSecret for public non-enclave hosts. secret otherwise - if (zone.system().isPublic() && !node.cloudAccount().isEnclave(zone)) { - String keyPem = secretStore.getSecret(athenzProviderServiceConfig.sisSecretName(), athenzProviderServiceConfig.sisSecretVersion()); - return KeyUtils.fromPemEncodedPrivateKey(keyPem); - } else { - return keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); - } - } - private String ztsUrl(Node node) { - // return sisUrl for public non-enclave hosts, ztsUrl otherwise - if (zone.system().isPublic() && !node.cloudAccount().isEnclave(zone)) { - return athenzProviderServiceConfig.sisUrl(); - } else { - return athenzProviderServiceConfig.ztsUrl(); - } - } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java index a349bddc76b..d8bbf743d8c 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java @@ -7,9 +7,6 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; @@ -55,32 +52,25 @@ public class InstanceValidator { private final KeyProvider keyProvider; private final SuperModelProvider superModelProvider; private final NodeRepository nodeRepository; - private final SecretStore secretStore; - private final String sisSecretName; @Inject public InstanceValidator(KeyProvider keyProvider, SuperModelProvider superModelProvider, NodeRepository nodeRepository, - AthenzProviderServiceConfig config, - SecretStore secretStore) { - this(keyProvider, superModelProvider, nodeRepository, new IdentityDocumentSigner(), new AthenzService(config.tenantService()), secretStore, config.sisSecretName()); + AthenzProviderServiceConfig config) { + this(keyProvider, superModelProvider, nodeRepository, new IdentityDocumentSigner(), new AthenzService(config.tenantService())); } public InstanceValidator(KeyProvider keyProvider, SuperModelProvider superModelProvider, NodeRepository nodeRepository, IdentityDocumentSigner identityDocumentSigner, - AthenzService tenantIdentity, - SecretStore secretStore, - String sisSecretName) { + AthenzService tenantIdentity){ this.keyProvider = keyProvider; this.superModelProvider = superModelProvider; this.nodeRepository = nodeRepository; this.signer = identityDocumentSigner; this.tenantDockerContainerIdentity = tenantIdentity; - this.secretStore = secretStore; - this.sisSecretName = sisSecretName; } public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { @@ -112,30 +102,16 @@ public class InstanceValidator { log.log(Level.FINE, () -> String.format("Validating instance %s.", providerUniqueId)); - // Find node matching vespa unique id - Node node = getNode(providerUniqueId); - - PublicKey publicKey = publicKey(node, signedIdentityDocument.signingKeyVersion()); + PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion()); if (! signer.hasValidSignature(signedIdentityDocument, publicKey)) { var msg = String.format("Instance %s has invalid signature.", providerUniqueId); throw new ValidationException(Level.SEVERE, () -> msg); } - validateAttributes(node, req, providerUniqueId); + validateAttributes(req, providerUniqueId); log.log(Level.FINE, () -> String.format("Instance %s is valid.", providerUniqueId)); } - private PublicKey publicKey(Node node, int version) { - // return sisSecret for public non-enclave hosts. - Zone zone = nodeRepository.zone(); - if (zone.system().isPublic() && !node.cloudAccount().isEnclave(zone)) { - String keyPem = secretStore.getSecret(sisSecretName, version); - return KeyUtils.extractPublicKey(KeyUtils.fromPemEncodedPrivateKey(keyPem)); - } else { - return keyProvider.getPublicKey(version); - } - } - // TODO Add actual validation. Cannot reuse isValidInstance as identity document is not part of the refresh request. // We'll have to perform some validation on the instance id and other fields of the attribute map. // Separate between tenant and node certificate as well. @@ -145,8 +121,7 @@ public class InstanceValidator { confirmation.provider, confirmation.attributes.get(SAN_DNS_ATTRNAME))); try { - VespaUniqueInstanceId vespaUniqueInstanceId = getVespaUniqueInstanceId(confirmation); - validateAttributes(getNode(vespaUniqueInstanceId), confirmation, vespaUniqueInstanceId); + validateAttributes(confirmation, getVespaUniqueInstanceId(confirmation)); return true; } catch (ValidationException e) { log.log(e.logLevel(), e.messageSupplier()); @@ -171,13 +146,24 @@ public class InstanceValidator { .orElse(null); } - private void validateAttributes(Node node, InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) + private void validateAttributes(InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) throws ValidationException { if(vespaUniqueInstanceId == null) { var msg = "Unable to find unique instance ID in refresh request: " + confirmation.toString(); throw new ValidationException(Level.WARNING, () -> msg); } + // Find node matching vespa unique id + Node node = nodeRepository.nodes().list().stream() + .filter(n -> n.allocation().isPresent()) + .filter(n -> nodeMatchesVespaUniqueId(n, vespaUniqueInstanceId)) + .findFirst() // Should be only one + .orElse(null); + if(node == null) { + var msg = "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId; + throw new ValidationException(Level.WARNING, () -> msg); + } + // Find list of ipaddresses List<InetAddress> ips = Optional.ofNullable(confirmation.attributes.get(SAN_IPS_ATTRNAME)) .map(s -> s.split(",")) @@ -213,20 +199,6 @@ public class InstanceValidator { } } - private Node getNode(VespaUniqueInstanceId vespaUniqueInstanceId) throws ValidationException { - // Find node matching vespa unique id - Node node = nodeRepository.nodes().list().stream() - .filter(n -> n.allocation().isPresent()) - .filter(n -> nodeMatchesVespaUniqueId(n, vespaUniqueInstanceId)) - .findFirst() // Should be only one - .orElse(null); - if(node == null) { - var msg = "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId; - throw new ValidationException(Level.WARNING, () -> msg); - } - return node; - } - private boolean nodeMatchesVespaUniqueId(Node node, VespaUniqueInstanceId vespaUniqueInstanceId) { return node.allocation().map(allocation -> allocation.membership().index() == vespaUniqueInstanceId.clusterIndex() && diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 231f22ac56b..531a815922b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -65,7 +65,7 @@ public class CertificateAuthorityApiHandler extends ThreadedHttpRequestHandler { super(ctx); this.secretStore = secretStore; this.certificates = certificates; - this.caPrivateKeySecretName = athenzProviderServiceConfig.sisSecretName(); + this.caPrivateKeySecretName = athenzProviderServiceConfig.secretName(); this.caCertificateSecretName = athenzProviderServiceConfig.caCertSecretName(); this.instanceValidator = instanceValidator; } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java index 8c575a6403b..fec03afab69 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java @@ -49,7 +49,6 @@ public class InstanceSerializer { private static final String IDD_IPADDRESSES_FIELD = "ip-addresses"; private static final String IDD_IDENTITY_TYPE_FIELD = "identity-type"; private static final String IDD_CLUSTER_TYPE_FIELD = "cluster-type"; - private static final String IDD_ZTS_URL_FIELD = "zts-url"; private static final ObjectMapper objectMapper = new ObjectMapper(); static { @@ -101,12 +100,10 @@ public class InstanceSerializer { IdentityType identityType = IdentityType.fromId(requireField(IDD_IDENTITY_TYPE_FIELD, root).asString()); var clusterTypeField = root.field(IDD_CLUSTER_TYPE_FIELD); var clusterType = clusterTypeField.valid() ? ClusterType.from(clusterTypeField.asString()) : null; - var ztsUrlField = root.field(IDD_ZTS_URL_FIELD); - var ztsUrl = ztsUrlField.valid() ? ztsUrlField.asString() : ""; return new SignedIdentityDocument(signature, (int)signingKeyVersion, providerUniqueId, athenzService, (int)documentVersion, - configserverHostname, instanceHostname, createdAt, ips, identityType, clusterType, ztsUrl); + configserverHostname, instanceHostname, createdAt, ips, identityType, clusterType); } private static Instant getJsr310Instant(double v) { diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java index 340be33c2a3..9205baff0fc 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -75,12 +74,11 @@ public class IdentityDocumentGeneratorTest { when(nodes.node(eq(parentHostname))).thenReturn(Optional.of(parentNode)); when(nodes.node(eq(containerHostname))).thenReturn(Optional.of(containerNode)); AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - SecretStoreMock secretStore = new SecretStoreMock(); String dnsSuffix = "vespa.dns.suffix"; AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix); IdentityDocumentGenerator identityDocumentGenerator = - new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider, secretStore); + new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); // Verify attributes diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java index 42d6b92dea1..a7947aff283 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java @@ -9,16 +9,9 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; @@ -26,19 +19,15 @@ import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.ValidationException; -import com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Nodes; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; import org.junit.jupiter.api.Test; -import java.security.PrivateKey; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -75,13 +64,11 @@ public class InstanceValidatorTest { private final AthenzService vespaTenantDomain = new AthenzService("vespa.vespa.tenant"); private final AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - private final SecretStoreMock secretStore = new SecretStoreMock(); - private final String sisSecretName = "sis-secret-name"; @Test void application_does_not_exist() { SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -89,7 +76,7 @@ public class InstanceValidatorTest { void application_does_not_have_domain_set() { SuperModelProvider superModelProvider = mockSuperModelProvider( mockApplicationInfo(applicationId, 5, Collections.emptyList())); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, new IdentityDocumentSigner(), vespaTenantDomain); assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -101,7 +88,7 @@ public class InstanceValidatorTest { SuperModelProvider superModelProvider = mockSuperModelProvider( mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -119,7 +106,7 @@ public class InstanceValidatorTest { mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); IdentityDocumentSigner signer = mock(IdentityDocumentSigner.class); when(signer.hasValidSignature(any(), any())).thenReturn(true); - InstanceValidator instanceValidator = new InstanceValidator(mock(KeyProvider.class), superModelProvider, mockNodeRepo(), signer, vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(mock(KeyProvider.class), superModelProvider, mockNodeRepo(), signer, vespaTenantDomain); assertTrue(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -127,7 +114,7 @@ public class InstanceValidatorTest { @Test void rejects_invalid_provider_unique_id_in_csr() { SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); VespaUniqueInstanceId tamperedId = new VespaUniqueInstanceId(0, "default", "instance", "app", "tenant", "us-north-1", "dev", IdentityType.NODE); instanceConfirmation.set("sanDNS", tamperedId.asDottedString() + ".instanceid.athenz.dev-us-north-1.vespa.yahoo.cloud"); @@ -137,7 +124,7 @@ public class InstanceValidatorTest { @Test void rejects_unknown_ips_in_csr() { NodeRepository nodeRepository = mockNodeRepo(); - InstanceValidator instanceValidator = new InstanceValidator(null, mockSuperModelProvider(), nodeRepository, null, vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, mockSuperModelProvider(), nodeRepository, null, vespaTenantDomain); InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); Set<String> nodeIp = nodeRepository.nodes().list().owner(applicationId).stream().findFirst() .map(Node::ipConfig) @@ -155,7 +142,7 @@ public class InstanceValidatorTest { var props = Map.of(SERVICE_PROPERTIES_DOMAIN_KEY, domain, SERVICE_PROPERTIES_SERVICE_KEY, service); var info = new ServiceInfo("serviceName", "type", List.of(), props, "confId", "hostName"); var provider = mockSuperModelProvider(mockApplicationInfo(applicationId, 5, List.of(info))); - var instanceValidator = new InstanceValidator(keyProvider, provider, mockNodeRepo(), new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); + var instanceValidator = new InstanceValidator(keyProvider, provider, mockNodeRepo(), new IdentityDocumentSigner(), vespaTenantDomain); var instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); instanceConfirmation.set("sanURI", "vespa://cluster-type/content"); var exception = assertThrows(ValidationException.class, () -> instanceValidator.validateInstance(instanceConfirmation)); @@ -168,7 +155,7 @@ public class InstanceValidatorTest { NodeRepository nodeRepository = mock(NodeRepository.class); Nodes nodes = mock(Nodes.class); when(nodeRepository.nodes()).thenReturn(nodes); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); List<Node> nodeList = createNodes(10); Node node = nodeList.get(0); @@ -183,7 +170,7 @@ public class InstanceValidatorTest { @Test void rejects_refresh_on_ip_mismatch() { NodeRepository nodeRepository = mockNodeRepo(); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); Set<String> nodeIp = nodeRepository.nodes().list().owner(applicationId).stream().findFirst() .map(Node::ipConfig) @@ -204,7 +191,7 @@ public class InstanceValidatorTest { Nodes nodes = mock(Nodes.class); when(nodeRepository.nodes()).thenReturn(nodes); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); + InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); List<Node> nodeList = createNodes(10); @@ -215,40 +202,11 @@ public class InstanceValidatorTest { } - @Test - public void uses_correct_keys_in_public() throws ValidationException { - var sisKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 2048); - secretStore.setSecret(sisSecretName, KeyUtils.toPem(sisKeyPair.getPrivate())); - - var props = Map.of(SERVICE_PROPERTIES_DOMAIN_KEY, domain, SERVICE_PROPERTIES_SERVICE_KEY, service); - var info = new ServiceInfo("serviceName", "type", List.of(), props, "confId", "hostName"); - var provider = mockSuperModelProvider(mockApplicationInfo(applicationId, 5, List.of(info))); - - { - // non Enclave hosts in public should use the SIS key - var nonEnclaveHosts = createNodes(10, Optional.empty()); - var instanceValidator = new InstanceValidator(keyProvider, provider, mockNodeRepo(nonEnclaveHosts, SystemName.PublicCd), new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); - var instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service, sisKeyPair.getPrivate()); - instanceValidator.validateInstance(instanceConfirmation); - } - { - // Enclave hosts should use the Vespa provider key - var enclaveHosts = createNodes(10, Optional.of(CloudAccount.from("123456789012"))); - var instanceValidator = new InstanceValidator(keyProvider, provider, mockNodeRepo(enclaveHosts, SystemName.PublicCd), new IdentityDocumentSigner(), vespaTenantDomain, secretStore, sisSecretName); - var instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service, keyProvider.getPrivateKey(0)); - instanceValidator.validateInstance(instanceConfirmation); - } - } - private NodeRepository mockNodeRepo() { - return mockNodeRepo(createNodes(10), SystemName.cd); - } - - private NodeRepository mockNodeRepo(List<Node> nodeList, SystemName systemName) { NodeRepository nodeRepository = mock(NodeRepository.class); Nodes nodes = mock(Nodes.class); when(nodeRepository.nodes()).thenReturn(nodes); - when(nodeRepository.zone()).thenReturn(new Zone(systemName, Environment.dev, RegionName.from("us-north-1"))); + List<Node> nodeList = createNodes(10); Node node = nodeList.get(0); nodeList = allocateNode(nodeList, node, applicationId); when(nodes.list()).thenReturn(NodeList.copyOf(nodeList)); @@ -257,11 +215,6 @@ public class InstanceValidatorTest { private InstanceConfirmation createRegisterInstanceConfirmation( ApplicationId applicationId, String domain, String service) { - return createRegisterInstanceConfirmation(applicationId, domain, service, keyProvider.getPrivateKey(0)); - } - - private InstanceConfirmation createRegisterInstanceConfirmation( - ApplicationId applicationId, String domain, String service, PrivateKey signingKey) { VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(0, "default", applicationId.instance().value(), applicationId.application().value(), applicationId.tenant().value(), "us-north-1", "dev", IdentityType.NODE); var domainService = new AthenzService(domain, service); var clock = Instant.now(); @@ -269,12 +222,11 @@ public class InstanceValidatorTest { var signature = new IdentityDocumentSigner() .generateSignature( vespaUniqueInstanceId, domainService, "localhost", "localhost", clock, Set.of(), - IdentityType.NODE, signingKey); + IdentityType.NODE, keyProvider.getPrivateKey(0)); SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( signature, 0, vespaUniqueInstanceId, domainService, 0, "localhost", "localhost", - clock, Collections.emptySet(), IdentityType.NODE, clusterType, "https://zts.url"); + clock, Collections.emptySet(), IdentityType.NODE, clusterType); return createInstanceConfirmation(vespaUniqueInstanceId, domain, service, signedIdentityDocument); - } private InstanceConfirmation createRefreshInstanceConfirmation(ApplicationId applicationId, String domain, String service, List<String> ips) { @@ -323,17 +275,11 @@ public class InstanceValidatorTest { } private List<Node> createNodes(int num) { - return createNodes(num, Optional.empty()); - } - - private List<Node> createNodes(int num, Optional<CloudAccount> cloudAccount) { MockNodeFlavors flavors = new MockNodeFlavors(); List<Node> nodeList = new ArrayList<>(); for (int i = 0; i < num; i++) { - Node.Builder builder = Node.create("foo" + i, new IP.Config(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), - "foo" + i, flavors.getFlavorOrThrow("default"), NodeType.tenant); - cloudAccount.ifPresent(builder::cloudAccount); - Node node = builder.build(); + Node node = Node.create("foo" + i, new IP.Config(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), + "foo" + i, flavors.getFlavorOrThrow("default"), NodeType.tenant).build(); nodeList.add(node); } return nodeList; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java index d880fd5220b..8112f5779e5 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java @@ -64,8 +64,6 @@ public class ContainerTester { " <serviceName>servicename</serviceName>\n" + " <secretName>secretname</secretName>\n" + " <secretVersion>0</secretVersion>\n" + - " <sisSecretName>secretname</sisSecretName>\n" + - " <sisSecretVersion>0</sisSecretVersion>\n" + " <caCertSecretName>vespa.external.ca.cert</caCertSecretName>\n" + " <certDnsSuffix>suffix</certDnsSuffix>\n" + " <ztsUrl>https://localhost:123/</ztsUrl>\n" + diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java index 02398b19627..ca624918beb 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.ca.restapi; -import com.yahoo.config.provision.Cloud; import com.yahoo.security.Pkcs10CsrUtils; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; @@ -49,8 +48,7 @@ public class InstanceSerializerTest { Instant.now().truncatedTo(ChronoUnit.MICROS), // Truncate to the precision given from EntityBindingsMapper.toAttestationData() Collections.emptySet(), IdentityType.NODE, - ClusterType.CONTAINER, - "https://zts.url"); + ClusterType.CONTAINER); var json = String.format("{\n" + " \"provider\": \"provider_prod_us-north-1\",\n" + diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java index 163db3e107e..4151c1f15d7 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; public class InstanceValidatorMock extends InstanceValidator { public InstanceValidatorMock() { - super(null, null, null, null, null, null,""); + super(null, null, null, null, null); } @Override diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 82842a8c28b..14b39867348 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -38,8 +38,8 @@ <aopalliance.version>1.0</aopalliance.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.13.4</jackson2.version> - <jackson-databind.version>2.13.4.2</jackson-databind.version> + <jackson2.version>2.14.2</jackson2.version> + <jackson-databind.version>2.14.2</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 5707206019f..f5ad1dce4e7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -106,6 +106,8 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { private final boolean includeSourceFiles; private final TransformerFactory transformerFactory; + private DeploymentSpec deploymentSpec = null; + /** Creates from a directory with source files included */ public static FilesApplicationPackage fromFile(File appDir) { return fromFile(appDir, false); @@ -580,6 +582,12 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { IOUtils.writeFile(metaFile, metaData.asJsonBytes()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + private void preprocessXML(File destination, File inputXml, Zone zone) throws IOException { if ( ! inputXml.exists()) return; try { @@ -589,10 +597,9 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { instance, zone.environment(), zone.region(), - getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) - .map(DeploymentInstanceSpec::tags) - .orElse(Tags.empty())) + getDeploymentSpec().instance(instance) + .map(DeploymentInstanceSpec::tags) + .orElse(Tags.empty())) .run(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java index 93e038c786a..19f1414cd10 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java @@ -110,10 +110,7 @@ public class HostedOverrideProcessorComplexTest { private void assertOverride(InstanceName instance, Environment environment, RegionName region, String expected) throws TransformerException { ApplicationPackage app = FilesApplicationPackage.fromFile(new File(servicesFile).getParentFile()); Document inputDoc = Xml.getDocument(app.getServices()); - Tags tags = app.getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance).map(DeploymentInstanceSpec::tags)) - .orElse(Tags.empty()); + Tags tags = app.getDeploymentSpec().instance(instance).map(DeploymentInstanceSpec::tags).orElse(Tags.empty()); Document newDoc = new OverrideProcessor(instance, environment, region, tags).process(inputDoc); assertEquals(expected, Xml.documentAsString(newDoc, true)); } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 6c83b2029ad..4742d275918 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -84,6 +84,7 @@ public class FilesApplicationPackageTest { assertFalse(new File(appDir, "deployment.xml").exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertFalse(app.getDeployment().isPresent()); + assertTrue(app.getDeploymentSpec().isEmpty()); } @Test @@ -93,6 +94,7 @@ public class FilesApplicationPackageTest { assertTrue(deployment.exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertTrue(app.getDeployment().isPresent()); + assertFalse(app.getDeploymentSpec().isEmpty()); assertFalse(app.getMajorVersion().isPresent()); assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(new FileReader(deployment))); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0ce09c454a0..b6a183c06ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -2,6 +2,7 @@ package com.yahoo.config.application.api; import com.yahoo.component.Version; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -128,20 +129,7 @@ public interface ApplicationPackage { /** Returns the major version this application is valid for, or empty if it is valid for all versions */ default Optional<Integer> getMajorVersion() { - if (getDeployment().isEmpty()) return Optional.empty(); - - Element deployElement = XML.getDocument(getDeployment().get()).getDocumentElement(); - if (deployElement == null) return Optional.empty(); - - String majorVersionString = deployElement.getAttribute("major-version"); - if (majorVersionString == null || majorVersionString.isEmpty()) - return Optional.empty(); - try { - return Optional.of(Integer.parseInt(majorVersionString)); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("major-version must be an integer number, not '" + majorVersionString + "'"); - } + return getDeploymentSpec().majorVersion(); } /** @@ -168,6 +156,19 @@ public interface ApplicationPackage { String getServicesSource(); Optional<Reader> getDeployment(); + + /** + * Returns the parsed deployment spec of this, + * without validating it, and without reparsing on each request. + */ + DeploymentSpec getDeploymentSpec(); + + default DeploymentSpec parseDeploymentSpec(boolean validate) { + return getDeployment() + .map(new DeploymentSpecXmlReader(validate)::read) + .orElse(DeploymentSpec.empty); + } + Optional<Reader> getValidationOverrides(); List<ComponentInfo> getComponentsInfo(Version vespaVersion); @@ -226,6 +227,7 @@ public interface ApplicationPackage { /** * Readers for all the schema files. + * * @return a collection of readers for schemas */ Collection<NamedReader> getSchemas(); @@ -235,10 +237,9 @@ public interface ApplicationPackage { * application package. This is the entry point for the multi environment application package support. This method * will not mutate the existing application package. * - * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove - * @param logger A {@link DeployLogger} to add output that will be returned to the user - * - * @return A new application package instance pointing to a new location + * @param zone a valid {@link Zone} instance, used to decide which parts of services to keep and remove + * @param logger a {@link DeployLogger} to add output that will be returned to the user + * @return a new application package instance pointing to a new location */ default ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException { throw new UnsupportedOperationException("This application package does not support preprocessing"); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java index 8f88b1ba74c..48464904f44 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java @@ -87,27 +87,18 @@ public class Bcp { public static class Group { private final List<RegionMember> members; - private final List<Endpoint> endpoints; + private final Set<RegionName> memberRegions; private final Duration deadline; public Group(List<RegionMember> members, Duration deadline) { - this(members, List.of(), deadline); - } - - public Group(List<RegionMember> members, List<Endpoint> endpoints, Duration deadline) { this.members = List.copyOf(members); - this.endpoints = endpoints; + this.memberRegions = members.stream().map(member -> member.region()).collect(Collectors.toSet()); this.deadline = deadline; } public List<RegionMember> members() { return members; } - /** - * Returns the endpoints defined in this. - * These will be added to instances during XML import post processing - * and should not otherwise be exposed from here. - */ - List<Endpoint> endpoints() { return endpoints; } + public Set<RegionName> memberRegions() { return memberRegions; } /** * Returns the max time until the other regions must be able to handle the additional traffic diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 41644ebc87d..699010417bf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -88,6 +88,8 @@ public class DeploymentSpec { validateBcp(); } + public boolean isEmpty() { return this == empty; } + /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ private void validateTotalDelay(List<Step> steps) { long totalDelaySeconds = steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 91cb2f2622e..d04bb7ecfe0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -519,7 +519,7 @@ public class DeploymentSpecXmlReader { } Duration deadline = XML.attribute("deadline", groupElement).map(value -> toDuration(value, "deadline")).orElse(Duration.ZERO); - groups.add(new Bcp.Group(regions, endpoints, deadline)); + groups.add(new Bcp.Group(regions, deadline)); } validateAndConsolidate(endpointsByZone, zoneEndpoints); return new Bcp(groups); diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index d9918168266..13d87b852e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -7,8 +7,11 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.vespa.model.VespaModel; +import java.time.Duration; +import java.util.Comparator; import java.util.stream.Stream; /** @@ -67,6 +70,18 @@ public final class ConfigModelContext { return ConfigModelContext.create(deployState, vespaModel, configModelRepoAdder, parent, producerId); } + /** Returns a cluster info builder pre-populated with info known in this context. */ + public ClusterInfo.Builder clusterInfo() { + var instance = getApplicationPackage().getDeploymentSpec().instance(properties().applicationId().instance()); + if ( ! instance.isPresent()) return new ClusterInfo.Builder(); + var maxDeadline = instance.get().bcp().groups().stream() + .filter(group -> group.memberRegions().contains(properties().zone().region())) + .map(group -> group.deadline()) + .min(Comparator.comparing(deadline -> deadline)) + .orElse(Duration.ofMinutes(0)); + return new ClusterInfo.Builder().bcpDeadline(maxDeadline); + } + /** * Create an application context from a parent producer and an id. * diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1813e183a60..b8d63ba3778 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -24,6 +24,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; @@ -77,7 +78,7 @@ public class DeployState implements ConfigDefinitionStore { private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; - private final Zone zone; + private final Zone zone; // TODO: Zone is set separately both here and in properties private final QueryProfiles queryProfiles; private final SemanticRules semanticRules; private final ImportedMlModels importedModels; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index b351073bd25..5ea22ee4d25 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -36,9 +36,6 @@ public class Hosts { for (Host host : hosts) hostsBuilder.put(host.hostname(), host); this.hosts = hostsBuilder.build(); - - // Don't limit zk connections on non-hosted systems - System.setProperty("zookeeper.vespa.clients", ""); } /** Throw IllegalArgumentException if host aliases breaks invariants */ diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index dd6087eefc7..41697e61bf2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -129,6 +129,8 @@ public class InMemoryProvisioner implements HostProvisioner { this.retiredHostNames = Set.of(retiredHostNames); } + public Provisioned provisioned() { return provisioned; } + /** May affect e.g. the number of nodes/cluster. */ public InMemoryProvisioner setEnvironment(Environment environment) { this.environment = environment; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index b5999c15fad..3b715c63105 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -57,12 +58,14 @@ public class MockApplicationPackage implements ApplicationPackage { private final List<String> schemas; private final Map<Path, MockApplicationFile> files; private final String schemaDir; - private final Optional<String> deploymentSpec; + private final Optional<String> deploymentSpecString; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; + private DeploymentSpec deploymentSpec = null; + protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, Map<Path, MockApplicationFile> files, String schemaDir, @@ -74,7 +77,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.schemas = schemas; this.files = files; this.schemaDir = schemaDir; - this.deploymentSpec = Optional.ofNullable(deploymentSpec); + this.deploymentSpecString = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), @@ -102,6 +105,12 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + + @Override public Reader getHosts() { if (hostsS == null) return null; return new StringReader(hostsS); @@ -183,7 +192,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Optional<Reader> getDeployment() { - return deploymentSpec.map(StringReader::new); + return deploymentSpecString.map(StringReader::new); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index 7dbab87fac0..047a6ef9bd5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -34,8 +34,6 @@ public final class Host extends TreeConfigProducer<AnyConfigProducer> implements Objects.requireNonNull(hostname, "The host name of a host cannot be null"); this.runsConfigServer = runsConfigServer; this.hostname = hostname; - if (parent instanceof HostSystem) - ((HostSystem)parent).checkName(hostname); } public static Host createConfigServerHost(HostSystem hostSystem, String hostname) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index b204aeaacf6..00a1078b294 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -54,8 +54,8 @@ public class HostSystem extends TreeConfigProducer<Host> { this.isHosted = isHosted; } - void checkName(String hostname) { - if (isHosted) return; // Done in node-repo instead + String checkHostname(String hostname) { + if (isHosted) return hostname; // Done in node-repo instead if (doCheckIp) { BiConsumer<Level, String> logFunction = deployLogger::logApplicationPackage; @@ -71,6 +71,7 @@ public class HostSystem extends TreeConfigProducer<Host> { logFunction.accept(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } } + return hostname; } @Override @@ -88,10 +89,10 @@ public class HostSystem extends TreeConfigProducer<Host> { } private HostResource addNewHost(HostSpec hostSpec) { - Host host = Host.createHost(this, hostSpec.hostname()); - HostResource hostResource = new HostResource(host, hostSpec); + String hostname = checkHostname(hostSpec.hostname()); + HostResource hostResource = new HostResource(Host.createHost(this, hostname), hostSpec); hostSpec.networkPorts().ifPresent(np -> hostResource.ports().addNetworkPorts(np)); - hostname2host.put(host.getHostname(), hostResource); + hostname2host.put(hostname, hostResource); return hostResource; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 567ccbfa88b..80000e54b1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -123,7 +123,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterId), context.getDeployLogger(), - false) + false, + context.clusterInfo().build()) .keySet(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java deleted file mode 100644 index 3491f219a8e..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.UserBindingPattern; -import org.w3c.dom.Element; - -/** - * @author gjoranv - * @since 5.1.6 - */ -public class DomClientProviderBuilder extends DomHandlerBuilder { - - public DomClientProviderBuilder(ApplicationContainerCluster cluster) { - super(cluster); - } - - @Override - protected Handler doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> parent, Element clientElement) { - Handler client = createHandler(clientElement); - - for (Element binding : XML.getChildren(clientElement, "binding")) - client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); - - for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) - client.addServerBindings(UserBindingPattern.fromPattern(XML.getValue(serverBinding))); - - DomComponentBuilder.addChildren(deployState, parent, clientElement, client); - - return client; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index b5fa451fa0b..c968e31325a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.component.Version; @@ -266,8 +267,9 @@ public class NodesSpecification { ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger, - boolean stateful) { - return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful); + boolean stateful, + ClusterInfo clusterInfo) { + return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful, clusterInfo); } public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, @@ -275,7 +277,8 @@ public class NodesSpecification { ClusterSpec.Id clusterId, ZoneEndpoint zoneEndpoint, DeployLogger logger, - boolean stateful) { + boolean stateful, + ClusterInfo info) { if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId) @@ -286,7 +289,7 @@ public class NodesSpecification { .loadBalancerSettings(zoneEndpoint) .stateful(stateful) .build(); - return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount), logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount, info), logger); } private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 8398df6f5ac..07e879f0e9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import ai.vespa.validation.Validation; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.builder.xml.XmlHelper; @@ -24,12 +23,8 @@ import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; import org.w3c.dom.Element; - -import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 3d9a8441ed5..0be3c825614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -26,9 +26,9 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; - private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; - private final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; + public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 29b1edc1397..19df9a4064f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -10,6 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; + /** * NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)} * @@ -53,7 +57,10 @@ public class PlatformBundles { public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( SEARCH_AND_DOCPROC_BUNDLE, "docprocs", - "linguistics-components" + "linguistics-components", + EVALUATION_BUNDLE_NAME, + INTEGRATION_BUNDLE_NAME, + ONNXRUNTIME_BUNDLE_NAME ); private static Set<Path> toBundlePaths(String... bundleNames) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index a639c158d62..36d34b99223 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; @@ -345,10 +346,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; - Optional<DeploymentSpec> deploymentSpec = app.getDeployment().map(DeploymentSpec::fromXml); + DeploymentSpec deploymentSpec = app.getDeploymentSpec(); if (deploymentSpec.isEmpty()) return; - for (var deprecatedElement : deploymentSpec.get().deprecatedElements()) { + for (var deprecatedElement : deploymentSpec.deprecatedElements()) { deployLogger.logApplicationPackage(WARNING, deprecatedElement.humanReadableString()); } @@ -358,8 +359,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getProperties().ztsUrl(), context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), - deploymentSpec.get()); - addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec.get()); + deploymentSpec); + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec); } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { @@ -863,10 +864,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { InstanceName instance = context.properties().applicationId().instance(); ZoneId zone = ZoneId.from(context.properties().zone().environment(), context.properties().zone().region()); - DeploymentSpec spec = context.getApplicationPackage().getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .orElse(DeploymentSpec.empty); - return spec.zoneEndpoint(instance, zone, cluster); + return context.getApplicationPackage().getDeploymentSpec().zoneEndpoint(instance, zone, cluster); } private static Map<String, String> getEnvironmentVariables(Element environmentVariables) { @@ -924,22 +922,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { // request just enough nodes to satisfy environment capacity requirement - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName())) - .vespaVersion(deployState.getWantedNodeVespaVersion()) - .dockerImageRepository(deployState.getWantedDockerImageRepo()) - .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + - " nodes in " + cluster); - ClusterResources resources = new ClusterResources(nodeCount, 1, NodeResources.unspecified()); - Capacity capacity = Capacity.from(resources, - resources, - IntRange.empty(), - false, - !deployState.getProperties().isBootstrap(), - context.getDeployState().getProperties().cloudAccount()); - var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); + deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + " nodes in " + cluster); + var nodesSpec = NodesSpecification.dedicated(nodeCount, context); + var hosts = nodesSpec.provision(hostSystem, + ClusterSpec.Type.container, + ClusterSpec.Id.from(cluster.getName()), + deployState.getDeployLogger(), + false, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { @@ -956,15 +947,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { - NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); - ClusterSpec.Id clusterId = ClusterSpec.Id.from(cluster.name()); - ZoneEndpoint zoneEndpoint = zoneEndpoint(context, clusterId); + var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, clusterId, - zoneEndpoint, + zoneEndpoint(context, clusterId), log, - getZooKeeper(containerElement) != null); + getZooKeeper(containerElement) != null, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } catch (IllegalArgumentException e) { @@ -998,7 +989,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { StorageGroup.provisionHosts(nodeSpecification, referenceId, cluster.getRoot().hostSystem(), - context.getDeployLogger()); + context); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 31ec764fbde..52b2ce06dfe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -187,10 +187,15 @@ public class StorageGroup { public static Map<HostResource, ClusterMembership> provisionHosts(NodesSpecification nodesSpecification, String clusterIdString, - HostSystem hostSystem, - DeployLogger logger) { + HostSystem hostSystem, + ConfigModelContext context) { ClusterSpec.Id clusterId = ClusterSpec.Id.from(clusterIdString); - return nodesSpecification.provision(hostSystem, ClusterSpec.Type.content, clusterId, logger, true); + return nodesSpecification.provision(hostSystem, + ClusterSpec.Type.content, + clusterId, + context.getDeployLogger(), + true, + context.clusterInfo().build()); } public static class Builder { @@ -203,7 +208,9 @@ public class StorageGroup { this.context = context; } - public StorageGroup buildRootGroup(DeployState deployState, RedundancyBuilder redundancyBuilder, ContentCluster owner) { + public StorageGroup buildRootGroup(DeployState deployState, + RedundancyBuilder redundancyBuilder, + ContentCluster owner) { try { if (owner.isHosted()) validateRedundancyAndGroups(deployState.zone().environment()); @@ -219,7 +226,7 @@ public class StorageGroup { GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null); StorageGroup storageGroup = owner.isHosted() - ? groupBuilder.buildHosted(deployState, owner, Optional.empty()) + ? groupBuilder.buildHosted(deployState, owner, Optional.empty(), context) : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), storageGroup.subgroups.size(), @@ -334,12 +341,18 @@ public class StorageGroup { * @param parent the parent storage group, or empty if this is the root group * @return the storage group build by this */ - public StorageGroup buildHosted(DeployState deployState, ContentCluster owner, Optional<GroupBuilder> parent) { + public StorageGroup buildHosted(DeployState deployState, + ContentCluster owner, + Optional<GroupBuilder> parent, + ConfigModelContext context) { if (storageGroup.getIndex() != null) throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : + provisionHosts(nodeRequirement.get(), + owner.getStorageCluster().getClusterName(), + owner.getRoot().hostSystem(), + context) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -362,7 +375,7 @@ public class StorageGroup { storageGroup.nodes.add(createStorageNode(deployState, owner, host.getKey(), storageGroup, host.getValue())); } for (GroupBuilder subGroup : subGroups) { - storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this))); + storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this), context)); } } return storageGroup; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 137e19e7d86..7f4fc4cd89d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -150,7 +150,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (e != null) setupDocumentProcessing(c, e); } else if (c.persistenceFactory != null) { - throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); + throw new IllegalArgumentException("The <documents> element is mandatory in content cluster '" + clusterId + "'"); } ModelElement tuning = contentElement.child("tuning"); @@ -333,7 +333,8 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger(), - true) + true, + context.clusterInfo().build()) .keySet(); admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState), hosts, diff --git a/config-model/src/test/derived/globalphase_onnx_inside/.gitignore b/config-model/src/test/derived/globalphase_onnx_inside/.gitignore new file mode 100644 index 00000000000..d9609d7b326 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/.gitignore @@ -0,0 +1 @@ +models.generated diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java index 05b8681b5fa..d92fea24d51 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java @@ -70,8 +70,6 @@ public class HostsXmlProvisionerTest { assertEquals(3, map.size()); assertCorrectNumberOfHosts(map, 3); assertTrue(map.keySet().containsAll(aliases)); - - assertEquals("", System.getProperty("zookeeper.vespa.clients")); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java new file mode 100644 index 00000000000..0abb153696c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -0,0 +1,85 @@ +package com.yahoo.vespa.model; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bratseth + */ +public class ClusterInfoTest { + + @Test + void bcp_deadline_is_passed_in_cluster_info() throws Exception { + var servicesXml = """ + <services version='1.0'> + <container id='testcontainer' version='1.0'> + <nodes count='3'/> + </container> + <content id='testcontent' version='1.0'> + <redundancy>2</redundancy> + <documents/> + </content> + </services> + """; + + var deploymentXml = """ + <deployment version='1.0'> + <prod> + <region>us-west-1</region> + <region>us-east-1</region> + </prod> + <bcp> + <group deadline='30m'> + <region fraction='0.5'>us-east-1</region> + <region>us-west-1</region> + </group> + <group> + <region fraction='0.5'>us-east-1</region> + </group> + </bcp> + </deployment> + """; + + var requestedInUsEast1 = requestedCapacityIn("us-east-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + + var requestedInUsWest1 = requestedCapacityIn("us-west-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + } + + private Map<ClusterSpec.Id, Capacity> requestedCapacityIn(String region, String servicesXml, String deploymentXml) throws Exception { + var applicationPackage = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withDeploymentSpec(deploymentXml) + .build(); + + var provisioner = new InMemoryProvisioner(10, true); + var deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from(region))) + .properties(new TestProperties().setHostedVespa(true) + .setZone(new Zone(Environment.prod, RegionName.from(region)))) + .modelHostProvisioner(provisioner) + .provisioned(provisioner.provisioned()) + .build(); + new VespaModel(new NullConfigModelRegistry(), deployState); + return deployState.provisioned().all(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java index fcc8c82a6e9..a8a063cb5fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; @@ -57,7 +58,8 @@ class CloudAccountChangeValidatorTest { IntRange.empty(), false, false, - Optional.of(cloudAccount).filter(account -> !account.isUnspecified())); + Optional.of(cloudAccount).filter(account -> !account.isUnspecified()), + ClusterInfo.empty()); } private static VespaModel model(Provisioned provisioned) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 6ac4dbd17e3..cad36f5574c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -835,7 +835,7 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); }); - assertTrue(exception.getMessage().contains("The specified content engine requires the <documents> element to be specified.")); + assertEquals("The <documents> element is mandatory in content cluster 'a'", exception.getMessage()); } private ProtonConfig getProtonConfig(ContentCluster content) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 2477d19d46c..c167c862ce4 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -20,6 +20,7 @@ public final class Capacity { private final boolean canFail; private final NodeType type; private final Optional<CloudAccount> cloudAccount; + private final ClusterInfo clusterInfo; private Capacity(ClusterResources min, ClusterResources max, @@ -27,7 +28,8 @@ public final class Capacity { boolean required, boolean canFail, NodeType type, - Optional<CloudAccount> cloudAccount) { + Optional<CloudAccount> cloudAccount, + ClusterInfo clusterInfo) { validate(min); validate(max); if (max.smallerThan(min)) @@ -42,6 +44,7 @@ public final class Capacity { this.canFail = canFail; this.type = type; this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.clusterInfo = clusterInfo; } private static void validate(ClusterResources resources) { @@ -77,12 +80,14 @@ public final class Capacity { return cloudAccount; } + public ClusterInfo clusterInfo() { return clusterInfo; } + public Capacity withLimits(ClusterResources min, ClusterResources max) { return withLimits(min, max, IntRange.empty()); } public Capacity withLimits(ClusterResources min, ClusterResources max, IntRange groupSize) { - return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount); + return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount, clusterInfo); } @Override @@ -98,25 +103,21 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources min, ClusterResources max) { - return from(min, max, IntRange.empty(), false, true, Optional.empty()); + return from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { return from(resources, required, canFail, NodeType.tenant); } - // TODO: Remove after February 2023 - public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty()); - } - - // TODO: Remove after February 2023 - public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount); + // TODO: Remove after March 2023 + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); } - public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount); + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, + Optional<CloudAccount> cloudAccount, ClusterInfo clusterInfo) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, clusterInfo); } /** Creates this from a node type */ @@ -125,7 +126,7 @@ public final class Capacity { } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { - return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty()); + return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty(), ClusterInfo.empty()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java new file mode 100644 index 00000000000..dc01c37f854 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -0,0 +1,40 @@ +package com.yahoo.config.provision; + +import java.time.Duration; + +/** + * Auxiliary information about a cluster, provided by the config model to the node repo during a + * capacity request. + * + * @author bratseth + */ +public class ClusterInfo { + + private static final ClusterInfo empty = new ClusterInfo.Builder().build(); + + private final Duration bcpDeadline; + + private ClusterInfo(Builder builder) { + this.bcpDeadline = builder.bcpDeadline; + } + + public Duration bcpDeadline() { return bcpDeadline; } + + public static ClusterInfo empty() { return empty; } + + public static class Builder { + + private Duration bcpDeadline = Duration.ofMinutes(0); + + public Builder bcpDeadline(Duration duration) { + this.bcpDeadline = duration; + return this; + } + + public ClusterInfo build() { + return new ClusterInfo(this); + } + + } + +} diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java index 89c0e98b076..a7614bbc016 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java @@ -21,7 +21,8 @@ public class CapacityTest { IntRange.empty(), false, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1, 2, 3, 4)), new ClusterResources(2, 2, new NodeResources(1, 2, 3, 4))); assertValidationFailure(new ClusterResources(4, 4, new NodeResources(1, 2, 3, 4)), @@ -41,7 +42,7 @@ public class CapacityTest { private void assertValidationFailure(ClusterResources min, ClusterResources max) { try { - Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty()); + Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); fail("Expected exception with min " + min + " and max " + max); } catch (IllegalArgumentException e) { diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index 1c04aa3eaa8..561ddbc078c 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib vespalog fnet diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp index b9641cc19e0..9cd8d5c985a 100644 --- a/configd/src/apps/sentinel/connectivity.cpp +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -75,10 +75,9 @@ void classifyConnFails(ConnectivityMap &connectivityMap, LOG_ASSERT(cmIter != connectivityMap.end()); OutwardCheckContext cornerContext(goodNeighborSpecs.size(), nameToCheck, portToCheck, rpcServer.orb()); ConnectivityMap cornerProbes; - int ping_timeout = 1000; + int ping_timeout = 1000 + 50 * goodNeighborSpecs.size(); for (const auto & hp : goodNeighborSpecs) { cornerProbes.try_emplace(hp.first, spec(hp), cornerContext, ping_timeout); - ping_timeout += 20; } cornerContext.latch.await(); size_t numReportsUp = 0; @@ -154,10 +153,9 @@ Connectivity::checkConnectivity(RpcServer &rpcServer) { rpcServer.getPort(), rpcServer.orb()); ConnectivityMap connectivityMap; - int ping_timeout = 1000; + int ping_timeout = 1000 + 50 * _checkSpecs.size(); for (const auto &host_and_port : _checkSpecs) { connectivityMap.try_emplace(host_and_port.first, spec(host_and_port), checkContext, ping_timeout); - ping_timeout += 20; } checkContext.latch.await(); classifyConnFails(connectivityMap, _checkSpecs, rpcServer); diff --git a/configd/src/apps/sentinel/report-connectivity.cpp b/configd/src/apps/sentinel/report-connectivity.cpp index 8417384dda9..d059980aae9 100644 --- a/configd/src/apps/sentinel/report-connectivity.cpp +++ b/configd/src/apps/sentinel/report-connectivity.cpp @@ -22,9 +22,9 @@ ReportConnectivity::ReportConnectivity(FRT_RPCRequest *req, int timeout_ms, FRT_ auto map = Connectivity::specsFrom(cfg.value()); LOG(debug, "making connectivity report for %zd peers", map.size()); _remaining = map.size(); + timeout_ms += 50 * map.size(); for (const auto & [ hostname, port ] : map) { _checks.emplace_back(std::make_unique<PeerCheck>(*this, hostname, port, orb, timeout_ms)); - timeout_ms += 20; } } else { _parentRequest->SetError(FRTE_RPC_METHOD_FAILED, "failed getting model config"); diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index c374e93904e..80b53d1dc0a 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib config_cloudconfig diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 4c9c74f9b8f..2131aa88d30 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -13,11 +13,6 @@ secretName string # Secret version secretVersion int -# Tempory resources -sisSecretName string default="" -sisSecretVersion int default=0 -sisUrl string default = "" - # Secret name of CA certificate caCertSecretName string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index d61590c2e84..ad5423f0a94 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -247,7 +247,7 @@ public class ModelContextImpl implements ModelContext { this.queryDispatchWarmup = flagValue(source, appId, version, PermanentFlags.QUERY_DISPATCH_WARMUP); this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); - this.enableGlobalPhase = flagValue(source, appId, version, PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW); + this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); } @Override public int heapSizePercentage() { return heapPercentage; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 454ab9e490d..b9118602058 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -155,7 +155,7 @@ public class FileDirectory extends AbstractComponent { } // update last modified time so that maintainer deleting unused file references considers this as recently used - existingFile.setLastModified(Clock.systemUTC().instant().toEpochMilli()); + destinationDir.setLastModified(Clock.systemUTC().instant().toEpochMilli()); log.log(Level.FINE, "Directory for file reference '" + fileReference.value() + "' already exists and has all content"); return false; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index ffafbb8827e..06c3aa5330e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -270,8 +270,7 @@ public class SessionPreparer { // Validate after doing our own preprocessing on these two files ApplicationMetaData meta = applicationPackage.getMetaData(); InstanceName instance = meta.getApplicationId().instance(); - Tags tags = applicationPackage.getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) + Tags tags = applicationPackage.getDeploymentSpec().instance(instance) .map(DeploymentInstanceSpec::tags) .orElse(Tags.empty()); if (servicesXml.exists()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index fbf14fbdb8c..6d6901136a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; @@ -55,6 +56,8 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { public static final String allocatedHostsNode = "allocatedHosts"; private final ApplicationMetaData metaData; + private DeploymentSpec deploymentSpec = null; + public ZKApplicationPackage(AddFileInterface fileManager, Curator curator, Path sessionPath, int maxNodeSize) { verifyAppPath(curator, sessionPath); zkApplication = new ZKApplication(curator, sessionPath, maxNodeSize); @@ -73,6 +76,12 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { return Optional.of(readAllocatedHosts()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + /** * Reads allocated hosts at the given node. * diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 8f11e171ebe..f3a2c42852f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -101,7 +101,9 @@ public class ZKApplicationPackageTest { assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertEquals(dockerImage, readInfo.getHosts().iterator().next().dockerImageRepo().get().asString()); assertTrue(zkApp.getDeployment().isPresent()); + assertFalse(zkApp.getDeploymentSpec().isEmpty()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); + assertEquals("mydisc", zkApp.getDeploymentSpec().requireInstance("default").globalServiceId().get()); } private void feed(com.yahoo.vespa.curator.Curator zk, File dirToFeed) throws IOException { diff --git a/configutil/CMakeLists.txt b/configutil/CMakeLists.txt index 4ec9ad13648..a1d699ae009 100644 --- a/configutil/CMakeLists.txt +++ b/configutil/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos config_cloudconfig vbench vespalib diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index d7600cac927..a95e0061992 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -243,8 +243,8 @@ <error-prone-annotations.version>2.18.0</error-prone-annotations.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.13.4</jackson2.version> - <jackson-databind.version>2.13.4.2</jackson-databind.version> + <jackson2.version>2.14.2</jackson2.version> + <jackson-databind.version>2.14.2</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java index 87213362acd..b72f81f1439 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -6,6 +6,7 @@ import ai.vespa.models.evaluation.Model; import com.yahoo.component.annotation.Inject; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.ranking.RankProfilesEvaluator.GlobalPhaseData; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.tensor.Tensor; @@ -23,8 +24,6 @@ public class GlobalPhaseRanker { private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); private final RankProfilesEvaluatorFactory factory; - private final Set<String> skipProcessing = new HashSet<>(); - private final Map<String, Supplier<FunctionEvaluator>> scorers = new HashMap<>(); @Inject public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) { @@ -33,11 +32,14 @@ public class GlobalPhaseRanker { } public void process(Query query, Result result, String schema) { - var functionEvaluatorSource = underlying(query, schema); - if (functionEvaluatorSource == null) { + var proxy = factory.proxyForSchema(schema); + String rankProfile = query.getRanking().getProfile(); + var optData = proxy.getGlobalPhaseData(rankProfile); + if (optData.isEmpty()) return; - } - var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments()); + GlobalPhaseData data = optData.get(); + var functionEvaluatorSource = data.functionEvaluatorSource(); + var prepared = findFromQuery(query, data.needInputs()); Supplier<Evaluator> supplier = () -> { var evaluator = functionEvaluatorSource.get(); var simple = new SimpleEvaluator(evaluator); @@ -46,9 +48,10 @@ public class GlobalPhaseRanker { } return simple; }; - // TODO need to get rerank-count somehow - int rerank = 7; - ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank); + int rerankCount = data.rerankCount(); + if (rerankCount < 0) + rerankCount = 100; + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerankCount); } record NameAndValue(String name, Tensor value) { } @@ -86,33 +89,4 @@ public class GlobalPhaseRanker { return result; } - private Supplier<FunctionEvaluator> underlying(Query query, String schema) { - String rankProfile = query.getRanking().getProfile(); - String key = schema + " with rank profile " + rankProfile; - if (skipProcessing.contains(key)) { - return null; - } - Supplier<FunctionEvaluator> supplier = scorers.get(key); - if (supplier != null) { - return supplier; - } - try { - var proxy = factory.proxyForSchema(schema); - var model = proxy.modelForRankProfile(rankProfile); - supplier = () -> model.evaluatorOf("globalphase"); - if (supplier.get() == null) { - supplier = null; - } - } catch (IllegalArgumentException e) { - logger.info("no global-phase for " + key + " because: " + e.getMessage()); - supplier = null; - } - if (supplier == null) { - skipProcessing.add(key); - } else { - scorers.put(key, supplier); - } - return supplier; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java index ccb9b9837fe..2ca91a3ea91 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -14,6 +14,13 @@ import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.logging.Logger; + /** * proxy for model-evaluation components * @author arnej @@ -22,6 +29,7 @@ import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; public class RankProfilesEvaluator extends AbstractComponent { private final ModelsEvaluator evaluator; + private static final Logger logger = Logger.getLogger(RankProfilesEvaluator.class.getName()); @Inject public RankProfilesEvaluator( @@ -37,6 +45,7 @@ public class RankProfilesEvaluator extends AbstractComponent { expressionsConfig, onnxModelsConfig, fileAcquirer); + extractGlobalPhaseData(rankProfilesConfig); } public Model modelForRankProfile(String rankProfile) { @@ -50,4 +59,39 @@ public class RankProfilesEvaluator extends AbstractComponent { public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { return modelForRankProfile(rankProfile).evaluatorOf(functionName); } + + static record GlobalPhaseData(Supplier<FunctionEvaluator> functionEvaluatorSource, + int rerankCount, + List<String> needInputs) {} + + private Map<String, GlobalPhaseData> profilesWithGlobalPhase = new HashMap<>(); + + Optional<GlobalPhaseData> getGlobalPhaseData(String rankProfile) { + return Optional.ofNullable(profilesWithGlobalPhase.get(rankProfile)); + } + + private void extractGlobalPhaseData(RankProfilesConfig rankProfilesConfig) { + for (var rp : rankProfilesConfig.rankprofile()) { + String name = rp.name(); + Supplier<FunctionEvaluator> functionEvaluatorSource = null; + int rerankCount = -1; + List<String> needInputs = null; + + for (var prop : rp.fef().property()) { + if (prop.name().equals("vespa.globalphase.rerankcount")) { + rerankCount = Integer.valueOf(prop.value()); + } + if (prop.name().equals("vespa.rank.globalphase")) { + var model = modelForRankProfile(name); + functionEvaluatorSource = () -> model.evaluatorOf("globalphase"); + var evaluator = functionEvaluatorSource.get(); + needInputs = List.copyOf(evaluator.function().arguments()); + } + } + if (functionEvaluatorSource != null && needInputs != null) { + profilesWithGlobalPhase.put(name, new GlobalPhaseData(functionEvaluatorSource, rerankCount, needInputs)); + } + } + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java index 7b209d231c4..796ce5da449 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java @@ -20,14 +20,16 @@ public class NodeFilter { private final boolean includeDeprovisioned; private final Set<Node.State> states; private final Set<HostName> hostnames; + private final Set<HostName> parentHostnames; private final Set<ApplicationId> applications; private NodeFilter(boolean includeDeprovisioned, Set<Node.State> states, Set<HostName> hostnames, - Set<ApplicationId> applications) { + Set<HostName> parentHostnames, Set<ApplicationId> applications) { this.includeDeprovisioned = includeDeprovisioned; // Uses Guava Set to preserve insertion order this.states = ImmutableSet.copyOf(Objects.requireNonNull(states)); this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames)); + this.parentHostnames = ImmutableSet.copyOf(Objects.requireNonNull(parentHostnames)); this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications)); if (!includeDeprovisioned && states.contains(Node.State.deprovisioned)) { throw new IllegalArgumentException("Must include deprovisioned nodes when matching deprovisioned state"); @@ -46,12 +48,16 @@ public class NodeFilter { return hostnames; } + public Set<HostName> parentHostnames() { + return parentHostnames; + } + public Set<ApplicationId> applications() { return applications; } public NodeFilter includeDeprovisioned(boolean includeDeprovisioned) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter states(Node.State... states) { @@ -59,7 +65,7 @@ public class NodeFilter { } public NodeFilter states(Set<Node.State> states) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter hostnames(HostName... hostnames) { @@ -67,7 +73,15 @@ public class NodeFilter { } public NodeFilter hostnames(Set<HostName> hostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + } + + public NodeFilter parentHostnames(HostName... parentHostnames) { + return parentHostnames(ImmutableSet.copyOf(parentHostnames)); + } + + public NodeFilter parentHostnames(Set<HostName> parentHostnames) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter applications(ApplicationId... applications) { @@ -75,12 +89,12 @@ public class NodeFilter { } public NodeFilter applications(Set<ApplicationId> applications) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } /** A filter which matches all nodes, except deprovisioned ones */ public static NodeFilter all() { - return new NodeFilter(false, Set.of(), Set.of(), Set.of()); + return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 45b3e4ef5dd..da0fa890960 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -192,17 +192,15 @@ public class VcmrMaintainer extends ControllerMaintainer { } if (shouldRetire(changeRequest, hostAction)) { - if (!node.wantToRetire()) { + if (!wantToRetireRecursive(zoneId, node)) { LOG.info(Text.format("Retiring %s due to %s", node.hostname().value(), changeRequest.getChangeRequestSource().getId())); // TODO: Remove try/catch once retirement is stabilized try { setWantToRetire(zoneId, node, true); } catch (Exception e) { LOG.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); - // Check if retirement actually failed - if (!nodeRepository.getNode(zoneId, node.hostname().value()).wantToRetire()) { - return hostAction; - } + // Will retry next maintenance run + return hostAction; } } return hostAction.withState(State.RETIRING); @@ -225,6 +223,13 @@ public class VcmrMaintainer extends ControllerMaintainer { return hostAction; } + // Determines if a host and all its children are retiring + private boolean wantToRetireRecursive(ZoneId zoneId, Node node) { + var children = nodeRepository.list(zoneId, NodeFilter.all().parentHostnames(node.hostname())); + return node.wantToRetire() && + children.stream().allMatch(Node::wantToRetire); + } + // Dirty host iff the parked host was retired by this maintainer private void recycleNode(ZoneId zoneId, Node node, HostAction hostAction) { if (hostAction.getState() == State.RETIRED && diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index 1c76f58a6b2..82dc333d178 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -9,6 +9,7 @@ import com.yahoo.text.Text; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; @@ -76,7 +77,7 @@ public class Notifier { } private boolean dispatchEnabled(NotificationSource source) { - return Flags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) + return PermanentFlags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) .with(FetchVector.Dimension.TENANT_ID, source.tenant().value()) .value(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7fcad017569..6d3c15c2d57 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -596,7 +596,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var mergedAddress = updateTenantInfoAddress(inspector.field("address"), info.address()); var mergedInfo = info - .withName(getString(inspector.field("tenant").field("name"), info.name())) + .withName(getString(inspector.field("tenant").field("company"), info.name())) .withWebsite(getString(inspector.field("tenant").field("website"), info.website())) .withContact(mergedContact) .withAddress(mergedAddress); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 834777abb62..04da0105e99 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -12,6 +12,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; @@ -79,7 +80,7 @@ public class NotificationsDbTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); private final MockMailer mailer = new MockMailer(); - private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.NOTIFICATION_DISPATCH_FLAG.id(), true); + private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource)); @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index 0c031a13e6f..96edba27c6f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -58,7 +59,7 @@ public class NotifierTest { @Test void dispatch() throws IOException { var mailer = new MockMailer(); - var flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.NOTIFICATION_DISPATCH_FLAG.id(), true); + var flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); var notifier = new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource); var notification = new Notification(Instant.now(), Notification.Type.testPackage, Notification.Level.warning, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 8a37bb560e2..41622e669e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -77,7 +77,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .roles(Set.of(Role.administrator(tenantName))); tester.assertResponse(updateRequest, "{\"message\":\"Tenant info updated\"}", 200); - tester.assertResponse(request, "{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\",\"emailVerified\":false},\"tenant\":{\"company\":\"\",\"website\":\"https://example.com/\"}}", 200); + tester.assertResponse(request, "{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\",\"emailVerified\":false},\"tenant\":{\"company\":\"Scoober, Inc.\",\"website\":\"https://example.com/\"}}", 200); } @Test diff --git a/dist/vespa.spec b/dist/vespa.spec index e880734979a..58c2a18d3c1 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -689,7 +689,6 @@ fi %endif %dir %{_prefix} %dir %{_prefix}/lib64 -%{_prefix}/lib64/libfastos.so %{_prefix}/lib64/libfnet.so %{_prefix}/lib64/libvespadefaults.so %{_prefix}/lib64/libvespalib.so @@ -701,7 +700,6 @@ fi %endif %dir %{_prefix} %{_prefix}/lib64 -%exclude %{_prefix}/lib64/libfastos.so %exclude %{_prefix}/lib64/libfnet.so %exclude %{_prefix}/lib64/libvespadefaults.so %exclude %{_prefix}/lib64/libvespalib.so diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index 88dbe2816d9..e1e4d8ff5cc 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java index 199ca199667..dc5cf609381 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java @@ -94,11 +94,10 @@ public class BoolFieldValue extends FieldValue { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof BoolFieldValue)) return false; + if ( ! (o instanceof BoolFieldValue other)) return false; if ( ! super.equals(o)) return false; - BoolFieldValue that = (BoolFieldValue) o; - return (value == that.value); + return (value == other.value); } @Override diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index 9261bcf9114..beeda4afeb4 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog config_cloudconfig vespalib diff --git a/fastos/.gitignore b/fastos/.gitignore deleted file mode 100644 index 54e2680a6d8..00000000000 --- a/fastos/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*.ilk -*.pdb -.Build_completed -.Dist_completed -.Install_completed -.PreBuild_completed -bin -include -lib -update.log -Makefile diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt deleted file mode 100644 index 60813c569e4..00000000000 --- a/fastos/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - LIBS - src/vespa/fastos - - TESTS -) diff --git a/fastos/OWNERS b/fastos/OWNERS deleted file mode 100644 index 912f61c59b8..00000000000 --- a/fastos/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -baldersheim -arnej27959 diff --git a/fastos/README b/fastos/README deleted file mode 100644 index ed9afabffb8..00000000000 --- a/fastos/README +++ /dev/null @@ -1,4 +0,0 @@ -Old OS abstraction layer - -obsolete, to be replaced with implementations using -standard C++14 threads and newer unix networking APIs diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/fastos/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/fastos/src/vespa/fastos/.gitignore b/fastos/src/vespa/fastos/.gitignore deleted file mode 100644 index 004799df5b4..00000000000 --- a/fastos/src/vespa/fastos/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -*.So -*.core -*.exe -*.ilk -*.pdb -.depend -.depend.NEW -Debug -Makefile -Makefile.factory -Makefile.overrides -Makefile.pre -Release -autoconf.h -config_command.bat -config_command.sh -fastconfig -fastconfig.exe -fastos.lib -fastosconfig.h -libfastos.a -makefeatures -makemake -processtest.log -test.txt -vc60.idb -vc60.pdb -/libfastos.so.5.1 diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt deleted file mode 100644 index 29810a4f296..00000000000 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastos_objects OBJECT - SOURCES -) - -vespa_add_library(fastos - SOURCES - $<TARGET_OBJECTS:fastos_objects> - INSTALL lib64 - DEPENDS - ${CMAKE_DL_LIBS} -) - -find_package(Threads REQUIRED) -target_link_libraries(fastos PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index ff287d221ec..3f1d78a66a0 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib APPS diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 163a68f9c98..2d5e3b32437 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_add_library(fbench_httpclient STATIC httpclient.cpp DEPENDS fbench_util - fastos ) diff --git a/fbench/src/test/CMakeLists.txt b/fbench/src/test/CMakeLists.txt index d13b6b82a81..c81d818ed06 100644 --- a/fbench/src/test/CMakeLists.txt +++ b/fbench/src/test/CMakeLists.txt @@ -26,6 +26,5 @@ vespa_add_executable(fbench_clientstatus_app TEST clientstatus.cpp DEPENDS fbench_util - fastos ) vespa_add_test(NAME fbench_clientstatus_app COMMAND fbench_clientstatus_app) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 13150f58ba3..cc18dc2bd84 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index dcb69d5f77b..7691cf4031f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -281,13 +281,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( - "dispatch-notifications", false, - List.of("enygaard"), "2022-05-02", "2023-03-01", - "Whether we should send notification for a given tenant", - "Takes effect immediately", - TENANT_ID); - public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, List.of("tokle"), "2022-05-09", "2023-03-31", @@ -345,13 +338,6 @@ public class Flags { "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); - public static final UnboundBooleanFlag VESPA_ATHENZ_PROVIDER = defineFeatureFlag( - "vespa-athenz-provider", false, - List.of("mortent"), "2023-02-22", "2023-05-01", - "Enable athenz provider in public systems", - "Takes effect on next config server container start", - ZONE_ID); - public static final UnboundBooleanFlag ENABLE_GLOBAL_PHASE = defineFeatureFlag( "enable-global-phase", false, List.of("arnej", "bjorncs"), "2023-02-28", "2024-01-10", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index d990ccebc3c..f128d13b7c4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -329,6 +329,13 @@ public class PermanentFlags { ZONE_ID ); + public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( + "dispatch-notifications", true, + "Whether we should send notification for a given tenant", + "Takes effect immediately", + TENANT_ID); + + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/jrt_test/CMakeLists.txt b/jrt_test/CMakeLists.txt index ea8c8b94faa..a678cbf112a 100644 --- a/jrt_test/CMakeLists.txt +++ b/jrt_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib fnet diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index d02b99a393a..5823ebf54a7 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/lowercasing_test/CMakeLists.txt b/lowercasing_test/CMakeLists.txt index b08a6ef350d..119209d4227 100644 --- a/lowercasing_test/CMakeLists.txt +++ b/lowercasing_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib searchlib diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index a3cc4cc7045..5272dd21a31 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -3,9 +3,9 @@ #[non-test] # Contains dependencies that are not used exclusively in 'test' scope aopalliance:aopalliance:1.0 -com.fasterxml.jackson.core:jackson-annotations:2.13.4 -com.fasterxml.jackson.core:jackson-core:2.13.4 -com.fasterxml.jackson.core:jackson-databind:2.13.4.2 +com.fasterxml.jackson.core:jackson-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:27.1-jre diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index d75c17d4a00..5b95d8635d4 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 6bd7d98e207..fc49dcc744c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -41,7 +41,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; @@ -190,9 +189,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - // Allow all zts hosts while removing SIS - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis + HostnameVerifier ztsHostNameVerifier = useInternalZts + ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) + : null; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withIdentityProvider(hostIdentityProvider).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.registerInstance( configserverIdentity, @@ -205,15 +206,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } - /** - * Return zts url from identity document, fallback to ztsEndpoint - */ - private URI ztsEndpoint(SignedIdentityDocument doc) { - return Optional.ofNullable(doc.ztsUrl()) - .filter(s -> !s.isBlank()) - .map(URI::create) - .orElse(ztsEndpoint); - } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, SignedIdentityDocument doc) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); @@ -225,9 +217,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .build(); try { - // Allow all zts hosts while removing SIS - HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(doc)).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { + // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis + HostnameVerifier ztsHostNameVerifier = useInternalZts + ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) + : null; + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { InstanceIdentity instanceIdentity = ztsClient.refreshInstance( configserverIdentity, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 79382a8bf5b..ba99219638b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -388,7 +388,7 @@ public final class Node implements Nodelike { return with(history.with(new History.Event(History.Event.Type.up, agent, instant))); } - /** Returns whether this node is down, according to its recorded 'down' and 'up' events */ + /** Returns true if we have positive evidence that this node is down. */ public boolean isDown() { Optional<Instant> downAt = history().event(History.Event.Type.down).map(History.Event::at); if (downAt.isEmpty()) return false; @@ -396,6 +396,14 @@ public final class Node implements Nodelike { return !history().hasEventAfter(History.Event.Type.up, downAt.get()); } + /** Returns true if we have positive evidence that this node is up. */ + public boolean isUp() { + Optional<Instant> upAt = history().event(History.Event.Type.up).map(History.Event::at); + if (upAt.isEmpty()) return false; + + return !history().hasEventAfter(History.Event.Type.down, upAt.get()); + } + /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */ public Node allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { return this.with(new Allocation(owner, membership, requestedResources, new Generation(0, 0), false)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 9506bba73e7..2cc43a1eb33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -77,6 +77,10 @@ public class Autoscaling { return peak.equals(Load.zero()); } + public boolean isPresent() { + return ! isEmpty(); + } + @Override public boolean equals(Object o) { if ( ! (o instanceof Autoscaling other)) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index 10f3b54dd45..e94ddd8e543 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -53,7 +53,7 @@ public class ClusterNodesTimeseries { public String description() { return "initial measurements: " + initialCount + - ", after warmpup: " + afterWarmupCount + + ", after warmup: " + afterWarmupCount + ", after stable: " + afterStableCount + ", final measurements: " + finalCount; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 3fdb088097c..69c03dbf6dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -83,7 +83,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Autoscaling autoscaling = null; if (cluster.target().resources().isEmpty() || current.equals(cluster.target().resources().get())) { autoscaling = autoscaler.autoscale(application.get(), cluster, clusterNodes); - if ( ! autoscaling.isEmpty()) // Ignore empties we'll get from servers recently started + if ( autoscaling.isPresent() || cluster.target().isEmpty()) // Ignore empty from recently started servers cluster = cluster.withTarget(autoscaling); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 94683d463af..781debe26a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -42,14 +42,13 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { @Override protected double maintain() { - return updateActiveNodeDownState(); + return updateNodeHealth(); } /** - * If the node is down (see {@link #allDown}), and there is no "down" history record, we add it. - * Otherwise we remove any "down" history record. + * Update UP and DOWN node records for each node as they change. */ - private double updateActiveNodeDownState() { + private double updateNodeHealth() { var attempts = new MutableInteger(0); var failures = new MutableInteger(0); NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); @@ -59,7 +58,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { // Already correct record, nothing to do boolean isDown = allDown(serviceInstances); - if (isDown == node.get().isDown()) return; + if (isDown == node.get().isDown() && isDown != node.get().isUp()) return; // Lock and update status ApplicationId owner = node.get().allocation().get().owner(); @@ -82,7 +81,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { } /** - * Returns true if the node is considered bad: All monitored services services are down. + * Returns true if the node is considered bad: All monitored services are down. * If a node remains bad for a long time, the NodeFailer will try to fail the node. */ static boolean allDown(List<ServiceInstance> services) { @@ -110,7 +109,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { /** Clear down record for node, if any */ private void recordAsUp(Node node, Mutex lock) { - if (!node.isDown()) return; // already up: Don't change down timestamp + if (node.isUp()) return; // already up: Don't change up timestamp nodeRepository().nodes().write(node.upAt(clock().instant(), Agent.NodeHealthTracker), lock); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 66d1568262b..0a614cc9b2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ActivationContext; @@ -208,7 +209,8 @@ public class MockNodeRepository extends NodeRepository { IntRange.empty(), false, true, - Optional.empty()), + Optional.empty(), + ClusterInfo.empty()), null), app1Id, provisioner); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 11bc26d90a5..d69d9267cfd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -389,7 +390,7 @@ public class AutoscalingTest { var fixture = DynamicProvisioningTester.fixture() .awsProdSetup(true) .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty())) + .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty(), ClusterInfo.empty())) .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.4, 240); @@ -497,7 +498,7 @@ public class AutoscalingTest { var fixture = DynamicProvisioningTester.fixture() .awsProdSetup(true) .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty())) + .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty(), ClusterInfo.empty())) .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.9, 120); @@ -743,7 +744,7 @@ public class AutoscalingTest { new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); var fixture = DynamicProvisioningTester.fixture() .awsSetup(true, Environment.dev) - .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty())) + .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty(), ClusterInfo.empty())) .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); @@ -763,7 +764,8 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); var fixture = DynamicProvisioningTester.fixture() .hostCount(5) @@ -785,7 +787,8 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); var fixture = DynamicProvisioningTester.fixture() .hostCount(5) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 1e1ccf37c8c..40ca30d758e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -62,10 +63,10 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); // noop diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index 50e19b78905..880a69b61e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -467,7 +468,7 @@ public class HostCapacityMaintainerTest { ClusterSpec spec = ProvisioningTester.contentClusterSpec(); ClusterResources resources = new ClusterResources(2, 1, new NodeResources(16, 24, 100, 1)); CloudAccount cloudAccount0 = CloudAccount.from("000000000000"); - Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); List<HostSpec> prepared = provisioningTester.prepare(applicationId, spec, capacity0); // Hosts are provisioned in requested account @@ -477,7 +478,7 @@ public class HostCapacityMaintainerTest { // Redeployment in different account provisions a new set of hosts CloudAccount cloudAccount1 = CloudAccount.from("100000000000"); - Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); prepared = provisioningTester.prepare(applicationId, spec, capacity1); provisionHostsIn(cloudAccount1, 2, tester); assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 54062e966f1..491485b78fc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -199,6 +199,11 @@ public class NodeFailerTest { @Test public void node_failing() { NodeFailTester tester = NodeFailTester.withTwoApplications(6); + String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); + String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname(); + // No liveness evidence yet: + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isUp()); // For a day all nodes work so nothing happens for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) { @@ -208,10 +213,10 @@ public class NodeFailerTest { assertEquals(0, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isUp()); } - String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); - String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname(); tester.serviceMonitor.setHostDown(downHost1); tester.serviceMonitor.setHostDown(downHost2); // nothing happens the first 45 minutes @@ -221,12 +226,16 @@ public class NodeFailerTest { assertEquals(0, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isUp()); } tester.serviceMonitor.setHostUp(downHost1); // downHost2 should now be failed and replaced, but not downHost1 tester.clock.advance(Duration.ofDays(1)); tester.runMaintainers(); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isUp()); assertEquals(1, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(1, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 9c69543a9d6..2786da4b69e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; @@ -56,10 +57,10 @@ public class ScalingSuggestionsMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofHours(13)); Duration timeAdded = addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository()); @@ -97,7 +98,7 @@ public class ScalingSuggestionsMaintainerTest { maintainer.maintain(); var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggested().resources().get(); tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofDays(2)); addMeasurements(0.2f, 0.65f, 0.6f, 0, 500, app1, tester.nodeRepository()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index c791c7848d7..d7b5f30a9bc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -6,6 +6,7 @@ import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; @@ -326,7 +327,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_custom_settings() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -343,7 +344,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_changing_visibility() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -379,7 +380,7 @@ public class LoadBalancerProvisionerTest { ClusterResources resources = new ClusterResources(3, 1, nodeResources); CloudAccount cloudAccount0 = CloudAccount.empty; { - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); } LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); @@ -388,7 +389,7 @@ public class LoadBalancerProvisionerTest { // Changing account fails if there is an existing LB in the previous account. CloudAccount cloudAccount1 = CloudAccount.from("111111111111"); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); try { prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"))); fail("Expected exception"); diff --git a/parent/pom.xml b/parent/pom.xml index 273e9b0ef8f..f5bb4ab8855 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -468,6 +468,11 @@ <version>3.10.0</version> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-cbor</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> <groupId>com.github.cverges.expect4j</groupId> <artifactId>expect4j</artifactId> <version>1.6</version> diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index d87f172bfb5..072e273338b 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib document diff --git a/screwdriver.yaml b/screwdriver.yaml index d918078b80e..9d423cacfe7 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -222,6 +222,7 @@ jobs: echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)." exit 1 fi + meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo dnf install -y docker-ce docker-ce-cli containerd.io diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index d6c353e46c9..131460b0384 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos fnet vespalog vespalib diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index f22f57979b4..7ee6a02d03e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -94,9 +94,8 @@ class ConfigFile { using SP = std::shared_ptr<ConfigFile>; - vespalib::string _name; - time_t _modTime; - std::vector<char> _content; + vespalib::string _name; + std::vector<char> _content; public: ConfigFile(); @@ -111,7 +110,6 @@ public: ConfigFile::ConfigFile() : _name(), - _modTime(0), _content() { } @@ -120,7 +118,6 @@ ConfigFile::~ConfigFile() = default; ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &fullName) : _name(name), - _modTime(0), _content() { FastOS_File file; @@ -130,7 +127,6 @@ ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &ful int64_t fileSize = file.getSize(); _content.resize(fileSize); file.ReadBuf(_content.data(), fileSize); - _modTime = file.GetModificationTime(); } nbostream & @@ -138,7 +134,7 @@ ConfigFile::serialize(nbostream &stream) const { assert(strchr(_name.c_str(), '/') == nullptr); stream << _name; - stream << static_cast<int64_t>(_modTime);; + stream << int64_t(0ul); // Used to be modtime => unused uint32_t sz = _content.size(); stream << sz; stream.write(_content.data(), sz); @@ -150,9 +146,8 @@ ConfigFile::deserialize(nbostream &stream) { stream >> _name; assert(strchr(_name.c_str(), '/') == nullptr); - int64_t modTime; - stream >> modTime; - _modTime = modTime; + int64_t unused_modTime; + stream >> unused_modTime; uint32_t sz; stream >> sz; _content.resize(sz); diff --git a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp index 54d2cf6e1f5..97afce79861 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -164,7 +164,6 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, LOG(error, "Could not save schema to '%s'", schemaTmpName.c_str()); } - // XXX: FastOS layer violation FastOS_StatInfo statInfo; bool statres; statres = FastOS_File::Stat(schemaOrigName.c_str(), &statInfo); @@ -183,7 +182,6 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, } vespalib::File::sync(indexDir); } - // XXX: FastOS layer violation int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str()); if (renameres != 0) { int error = errno; diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 4ccab1a9380..44051a96578 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib vespaeval diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java index c568db34b4f..ffc05f966fa 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java @@ -32,9 +32,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Tenso */ public class ExpressionOptimizer { - private GBDTOptimizer gbdtOptimizer = new GBDTOptimizer(); - private GBDTForestOptimizer gbdtForestOptimizer = new GBDTForestOptimizer(); - private TensorOptimizer tensorOptimizer = new TensorOptimizer(); + private final GBDTOptimizer gbdtOptimizer = new GBDTOptimizer(); + private final GBDTForestOptimizer gbdtForestOptimizer = new GBDTForestOptimizer(); + private final TensorOptimizer tensorOptimizer = new TensorOptimizer(); /** Gets an optimizer instance used by this by class name, or null if the optimizer is not known */ public Optimizer getOptimizer(Class<?> clazz) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java index a091cc0287c..33b1824fdeb 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java @@ -59,7 +59,6 @@ public class TensorOptimizer extends Optimizer { * The ReduceJoin class determines whether or not the arguments are * compatible with the optimization. */ - @SuppressWarnings("unchecked") private ExpressionNode optimizeReduceJoin(ExpressionNode node) { if ( ! (node instanceof TensorFunctionNode)) { return node; diff --git a/searchsummary/src/tests/juniper/auxTest.cpp b/searchsummary/src/tests/juniper/auxTest.cpp index 8eda6a2132a..a43294e709f 100644 --- a/searchsummary/src/tests/juniper/auxTest.cpp +++ b/searchsummary/src/tests/juniper/auxTest.cpp @@ -11,9 +11,7 @@ LOG_SETUP(".auxtest"); #define COLOR_HIGH_ON "\e[1;31m" #define COLOR_HIGH_OFF "\e[0m" -#ifndef FASTOS_DEBUG static int debug_level = 0; -#endif bool color_highlight = false; bool verbose = false; diff --git a/searchsummary/src/tests/juniper/testenv.cpp b/searchsummary/src/tests/juniper/testenv.cpp index cc6a6458376..1f8f52d8766 100644 --- a/searchsummary/src/tests/juniper/testenv.cpp +++ b/searchsummary/src/tests/juniper/testenv.cpp @@ -26,11 +26,7 @@ TestEnv::TestEnv(int argc, char **argv, const char* propfile) : switch (c) { case 'd': -#ifdef FASTOS_DEBUG - debug_level = strtol(optarg, NULL, 0); -#else fprintf(stderr, "This version of Juniper compiled without debug\n"); -#endif break; case 'c': color_highlight = true; diff --git a/searchsummary/src/vespa/juniper/juniperdebug.h b/searchsummary/src/vespa/juniper/juniperdebug.h index d475134dae3..f6d4eda2c46 100644 --- a/searchsummary/src/vespa/juniper/juniperdebug.h +++ b/searchsummary/src/vespa/juniper/juniperdebug.h @@ -27,23 +27,10 @@ /* Logging to log object (juniperlog summary field) */ #define JL(level, stmt) do { if (_log_mask & level) { stmt; } } while (0) -#ifdef FASTOS_DEBUG -extern unsigned debug_level; -#define JD(level, stmt) do { if (debug_level & level) { stmt; } } while (0) -# warning "FASTOS_DEBUG is defined" - -/* Invariant checking */ - -#define JD_INVAR(level, condition, action, log) \ - do { if (!(condition)) { if (debug_level & level) { log; } action; } } while (0) -#else - #define JD_INVAR(level, condition, action, log) \ do { if (!(condition)) { action; } } while (0) #define JD(level, stmt) -#endif - template <class _container> void dump_list(_container& __c) { diff --git a/searchsummary/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index 202b96a442d..afda82c1110 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -26,13 +26,6 @@ bool AnalyseCompatible(Config* conf1, Config* conf2) void SetDebug(unsigned int mask) { -#ifdef FASTOS_DEBUG - if (mask & ~1 && mask != debug_level) - LOG(info, "Juniper debug mode enabled (0x%x)", mask); - else if (! (debug_level & ~1)) - LOG(info, "Juniper debug mode disabled (0x%x)", mask); - debug_level = mask; -#else // Make sure we do not get 200 of these warnings per query.. static bool warning_printed = false; if (mask && !warning_printed) @@ -41,7 +34,6 @@ void SetDebug(unsigned int mask) "Juniper debug mode requested in binary compiled without debug support!"); warning_printed = true; } -#endif } diff --git a/searchsummary/src/vespa/juniper/tokenizer.cpp b/searchsummary/src/vespa/juniper/tokenizer.cpp index 9253f81cf25..1b58be2d451 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.cpp +++ b/searchsummary/src/vespa/juniper/tokenizer.cpp @@ -40,7 +40,6 @@ void JuniperTokenizer::scan() { if (_registry == NULL) { // explicit prefetching seems to have negative effect with many threads - // FastOS_Prefetch::NT(const_cast<void *>((const void *)(src + 32))); src = _wordfolder->UCS4Tokenize(src, src_end, dst, dst_end, startpos, result_len); } else { const char * tmpSrc = _registry->tokenize(src, src_end, dst, dst_end, startpos, result_len); diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 4b7c12b0f31..1062a89f055 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos metrics config_cloudconfig configdefinitions diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 8b82bb251b2..009f8170669 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_library(storage_storageserver OBJECT documentapiconverter.cpp fnet_metrics_wrapper.cpp mergethrottler.cpp - messagesink.cpp opslogger.cpp priorityconverter.cpp rpcrequestwrapper.cpp diff --git a/storage/src/vespa/storage/storageserver/messagesink.cpp b/storage/src/vespa/storage/storageserver/messagesink.cpp deleted file mode 100644 index 94762e545d9..00000000000 --- a/storage/src/vespa/storage/storageserver/messagesink.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messagesink.h" -#include <vespa/storageapi/message/persistence.h> -#include <ostream> - -using std::shared_ptr; - -namespace storage { - -MessageSink::MessageSink() - : StorageLink("Message Sink") -{ -} - -MessageSink::~MessageSink() -{ - closeNextLink(); -} - -void -MessageSink::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "MessageSink"; -} - -namespace { -#if 0 - std::string getTimeString() { - char timeBuf[200]; - time_t tm; - struct tm tms; - time(&tm); - gmtime_r(&tm, &tms); - strftime(timeBuf, sizeof(timeBuf), "%Y-%m-%d:%H:%M:%S %Z", &tms); - return std::string(timeBuf); - } -#endif -} - -IMPL_MSG_COMMAND_H(MessageSink, Get) -{ - //LOG(event, "[%s] Get %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::GetReply(*cmd)); - rmsg->setResult(api::ReturnCode::NOT_IMPLEMENTED); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Put) -{ - //LOG(event, "[%s] Put %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::PutReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Remove) -{ - //LOG(event, "[%s] Remove %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::RemoveReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Revert) -{ - //LOG(event, "[%s] Revert %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::RevertReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/messagesink.h b/storage/src/vespa/storage/storageserver/messagesink.h deleted file mode 100644 index d98d0439b48..00000000000 --- a/storage/src/vespa/storage/storageserver/messagesink.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::MessageSink - * @ingroup storageserver - * - * @brief This class grabs persistence messages, and answers them without doing anything. - * - * @version $Id$ - */ - -#pragma once - -#include <vespa/storage/common/storagelink.h> - -namespace storage { - -class MessageSink : public StorageLink { -public: - explicit MessageSink(); - MessageSink(const MessageSink &) = delete; - MessageSink& operator=(const MessageSink &) = delete; - ~MessageSink(); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - -private: - DEF_MSG_COMMAND_H(Get); - DEF_MSG_COMMAND_H(Put); - DEF_MSG_COMMAND_H(Remove); - DEF_MSG_COMMAND_H(Revert); -}; - -} diff --git a/storage/src/vespa/storageframework/generic/component/component.h b/storage/src/vespa/storageframework/generic/component/component.h index 9a5e524e504..47469cce05d 100644 --- a/storage/src/vespa/storageframework/generic/component/component.h +++ b/storage/src/vespa/storageframework/generic/component/component.h @@ -45,12 +45,12 @@ * optimize clock fetching as we see fit later. * * - A thread pool is given. This makes us able to use a thread pool. - * (Allthough currently we don't really need a thread pool, as threads - * typically live for the whole lifetime of the server. But currently we are - * forced to use a thread pool due to fastos.) Another feature of this is - * that the thread interface has built in information needed to detect - * deadlocks and report status about thread behavior, such that deadlock - * detecting and thread status can be shown without the threads themselves + * (Allthough currently we don't really need a thread pool, as + * threads typically live for the whole lifetime of the + * server. Another feature of this is that the thread interface has + * built in information needed to detect deadlocks and report + * status about thread behavior, such that deadlock detecting and + * thread status can be shown without the threads themselves * depending on how this is done. * * - A memory manager may also be provided, allowing components to request diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index ee3335c4921..a2f9d0b776e 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos storage streamingvisitors diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 2c7f01ddf37..fede7087d8d 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog storage config_cloudconfig diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index 3fb5df8cd20..e78913262be 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib LIBS diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 0f8144b99e9..1276323f83b 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/vdstestlib/CMakeLists.txt b/vdstestlib/CMakeLists.txt index d0a921672c2..7f478989332 100644 --- a/vdstestlib/CMakeLists.txt +++ b/vdstestlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib TESTS diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 067e8a6b00f..9b7b666e353 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -58,7 +58,6 @@ public class EntityBindingsMapper { entity.ipAddresses(), IdentityType.fromId(entity.identityType()), Optional.ofNullable(entity.clusterType()).map(ClusterType::from).orElse(null), - entity.ztsUrl(), entity.unknownAttributes()); } @@ -75,7 +74,6 @@ public class EntityBindingsMapper { model.ipAddresses(), model.identityType().id(), Optional.ofNullable(model.clusterType()).map(ClusterType::toConfigValue).orElse(null), - model.ztsUrl(), model.unknownAttributes()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index eba89b72d87..49a39d25e87 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.yahoo.vespa.athenz.api.AthenzService; -import java.net.URL; import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -18,7 +17,7 @@ import java.util.Set; public record SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl, Map<String, Object> unknownAttributes) { + IdentityType identityType, ClusterType clusterType, Map<String, Object> unknownAttributes) { public SignedIdentityDocument { ipAddresses = Set.copyOf(ipAddresses); @@ -34,9 +33,9 @@ public record SignedIdentityDocument(String signature, int signingKeyVersion, Ve public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, AthenzService providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl) { + IdentityType identityType, ClusterType clusterType) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, Map.of()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, Map.of()); } public static final int DEFAULT_DOCUMENT_VERSION = 2; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index edbe032ec26..c37dd2f9147 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -17,7 +17,7 @@ import java.util.Set; public record SignedIdentityDocumentEntity( String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, String ztsUrl, Map<String, Object> unknownAttributes) { + String identityType, String clusterType, Map<String, Object> unknownAttributes) { @JsonCreator public SignedIdentityDocumentEntity(@JsonProperty("signature") String signature, @@ -30,10 +30,9 @@ public record SignedIdentityDocumentEntity( @JsonProperty("created-at") Instant createdAt, @JsonProperty("ip-addresses") Set<String> ipAddresses, @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType, - @JsonProperty("zts-url") String ztsUrl) { + @JsonProperty("cluster-type") String clusterType) { this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, new HashMap<>()); + instanceHostname, createdAt, ipAddresses, identityType, clusterType, new HashMap<>()); } @JsonProperty("signature") @Override public String signature() { return signature; } @@ -47,7 +46,6 @@ public record SignedIdentityDocumentEntity( @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } @JsonProperty("identity-type") @Override public String identityType() { return identityType; } @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } - @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index 2a68f6fd231..f8c119190a6 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -30,7 +30,6 @@ class EntityBindingsMapperTest { "ip-addresses": [], "identity-type": "node", "cluster-type": "admin", - "zts-url": "https://zts.url/", "unknown-string": "string-value", "unknown-object": { "member-in-unknown-object": 123 } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 72798b03fa8..0b8ff4277f1 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -36,7 +36,6 @@ public class IdentityDocumentSignerTest { private static final Instant createdAt = Instant.EPOCH; private static final HashSet<String> ipAddresses = new HashSet<>(Arrays.asList("1.2.3.4", "::1")); private static final ClusterType clusterType = ClusterType.CONTAINER; - private static final String ztsUrl = "https://foo"; @Test void generates_and_validates_signature() { @@ -47,27 +46,27 @@ public class IdentityDocumentSignerTest { SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl); + instanceHostname, createdAt, ipAddresses, identityType, clusterType); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } @Test - void ignores_cluster_type_and_zts_url() { + void ignores_cluster_type() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); String signature = signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, ipAddresses, identityType, keyPair.getPrivate()); - var docWithoutIgnoredFields = new SignedIdentityDocument( + var docWithoutClusterType = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null, null); - var docWithIgnoredFields = new SignedIdentityDocument( + instanceHostname, createdAt, ipAddresses, identityType, null); + var docWithClusterType = new SignedIdentityDocument( signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl); + instanceHostname, createdAt, ipAddresses, identityType, clusterType); - assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); - assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); + assertTrue(signer.hasValidSignature(docWithoutClusterType, keyPair.getPublic())); + assertEquals(docWithClusterType.signature(), docWithoutClusterType.signature()); } }
\ No newline at end of file diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 5591db26054..d5166a7e22f 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -14,15 +14,15 @@ com.amazonaws:aws-java-sdk-ssm:1.12.331 com.amazonaws:aws-java-sdk-sts:1.12.331 com.amazonaws:jmespath-java:1.12.331 com.auth0:java-jwt:3.10.0 -com.fasterxml.jackson.core:jackson-annotations:2.13.4 -com.fasterxml.jackson.core:jackson-core:2.13.4 -com.fasterxml.jackson.core:jackson-databind:2.13.4.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.6 -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.4 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.4 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.4 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.4 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.4 +com.fasterxml.jackson.core:jackson-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.2 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.2 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.18.0 diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 912b35fa763..6e82b83517e 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos configdefinitions config_cloudconfig vespalog diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 8f1b687449a..8175e75875a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,7 +6,6 @@ endif() vespa_define_module( DEPENDS - fastos vespalog EXTERNAL_DEPENDS diff --git a/vespalib/src/vespa/fastlib/io/bufferedfile.cpp b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp index 31ca735a0bb..aecf08edf6b 100644 --- a/vespalib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp @@ -107,13 +107,6 @@ Fast_BufferedFile::Sync() return _file->Sync(); } -time_t -Fast_BufferedFile::GetModificationTime() -{ - time_t retval = _file->GetModificationTime(); - return retval; -} - void Fast_BufferedFile::EnableDirectIO() { diff --git a/vespalib/src/vespa/fastlib/io/bufferedfile.h b/vespalib/src/vespa/fastlib/io/bufferedfile.h index 48f90262ad9..2a5e0ec7535 100644 --- a/vespalib/src/vespa/fastlib/io/bufferedfile.h +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.h @@ -172,12 +172,7 @@ public: * Force completion of pending disk writes (flush cache). */ [[nodiscard]] bool Sync() override; - /** - * Get the time the file was last modified. - * - * @return time_t The last modification time. - */ - time_t GetModificationTime() override; + /** * Turn on direct IO. */ diff --git a/vespalib/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h index 146127d4fe6..371a43e27b6 100644 --- a/vespalib/src/vespa/fastos/file.h +++ b/vespalib/src/vespa/fastos/file.h @@ -10,7 +10,7 @@ #pragma once -#include <cstdint> +#include <vespa/vespalib/util/time.h> #include <string> #define FASTOS_PREFIX(a) FastOS_##a @@ -101,12 +101,6 @@ public: void setFAdviseOptions(int options) { _fAdviseOptions = options; } /** - * Return path separator string. This will yield "/" on UNIX systems. - * @return pointer to path separator character string - */ - static const char *GetPathSeparator() { return "/"; } - - /** * Constructor. A filename could be supplied at this point, or specified * later using @ref SetFileName() or @ref Open(). * @param filename a filename (optional) @@ -344,12 +338,6 @@ public: int64_t getSize() const { return const_cast<FastOS_FileInterface *>(this)->GetSize(); } /** - * Return the time when file was last modified. - * @return time of last modification - */ - virtual time_t GetModificationTime() = 0; - - /** * Delete the file. This method requires that the file is * currently not opened. * @return Boolean success/failure @@ -413,10 +401,6 @@ public: bool useSyncWrites() const { return _syncWritesEnabled; } - /** - * Set the write chunk size used in WriteBuf. - */ - void setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; } size_t getChunkSize() const { return _chunkSize; } /** @@ -652,14 +636,12 @@ public: */ class FastOS_DirectoryScanInterface { -private: - FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&); - FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); - protected: std::string _searchPath; public: + FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&) = delete; + FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&) = delete; /** * Constructor. @@ -677,13 +659,6 @@ public: virtual ~FastOS_DirectoryScanInterface(); /** - * Get search path. - * This is an internal copy of the path specified in the constructor. - * @return Search path string. - */ - const char *GetSearchPath () { return _searchPath.c_str(); } - - /** * Read the next entry in the directory scan. Failure indicates * that there are no more entries. If the call is successful, * attributes for the entry can be read with @ref IsDirectory(), @@ -726,14 +701,6 @@ public: * @return A pointer to the recently read directory entry. */ virtual const char *GetName() = 0; - - /** - * Check whether the creation of a directory scan succeeded or - * failed (e.g. due to resource constraints). - * - * return True if the directory scan is valid. - */ - virtual bool IsValidScan() const = 0; }; #ifdef __linux__ diff --git a/vespalib/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp index 7c4cde19125..417129c99b3 100644 --- a/vespalib/src/vespa/fastos/unix_file.cpp +++ b/vespalib/src/vespa/fastos/unix_file.cpp @@ -344,21 +344,6 @@ FastOS_UNIX_File::GetSize() return fileSize; } - -time_t -FastOS_UNIX_File::GetModificationTime() -{ - struct stat stbuf{}; - assert(IsOpened()); - - int res = fstat(_filedes, &stbuf); - assert(res == 0); - (void) res; - - return stbuf.st_mtime; -} - - bool FastOS_UNIX_File::Delete(const char *name) { @@ -581,10 +566,3 @@ FastOS_UNIX_DirectoryScan::GetName() return static_cast<const char *>(_dp->d_name); } - - -bool -FastOS_UNIX_DirectoryScan::IsValidScan() const -{ - return _dir != nullptr; -} diff --git a/vespalib/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h index 31e45f8d2fa..3d1f6b9db3f 100644 --- a/vespalib/src/vespa/fastos/unix_file.h +++ b/vespalib/src/vespa/fastos/unix_file.h @@ -82,7 +82,6 @@ public: bool SetPosition(int64_t desiredPosition) override; int64_t GetPosition() override; int64_t GetSize() override; - time_t GetModificationTime() override; bool Delete() override; [[nodiscard]] bool Sync() override; bool SetSize(int64_t newSize) override; @@ -103,9 +102,6 @@ public: class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface { private: - FastOS_UNIX_DirectoryScan(const FastOS_UNIX_DirectoryScan&); - FastOS_UNIX_DirectoryScan& operator=(const FastOS_UNIX_DirectoryScan&); - bool _statRun; bool _isDirectory; bool _isRegular; @@ -126,5 +122,4 @@ public: bool IsDirectory() override; bool IsRegular() override; const char *GetName() override; - bool IsValidScan() const override; }; diff --git a/vespalog/src/test/threads/CMakeLists.txt b/vespalog/src/test/threads/CMakeLists.txt index 00de16ff005..19fe2511025 100644 --- a/vespalog/src/test/threads/CMakeLists.txt +++ b/vespalog/src/test/threads/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(vespalog_threads_test_app TEST testthreads.cpp DEPENDS vespalog - fastos ) vespa_add_test(NAME vespalog_threads_test_app COMMAND vespalog_threads_test_app vespa.log ENVIRONMENT "VESPA_LOG_TARGET=file:vespa.log") diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt index dbeba17334b..82536b1f4e9 100644 --- a/vespalog/src/vespa/log/CMakeLists.txt +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -18,3 +18,4 @@ vespa_add_library(vespalog reject-filter.cpp INSTALL lib64 ) +target_link_libraries(vespalog PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index dd7c92a4dac..f3ce9f70f45 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -8,12 +8,15 @@ vespa_add_executable(vespamalloc_testatomic_app TEST vespamalloc_util EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} + dl ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) vespa_add_executable(vespamalloc_new_test_app TEST SOURCES new_test.cpp + EXTERNAL_DEPENDS + dl ) vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) |