diff options
25 files changed, 660 insertions, 158 deletions
diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index 739dc79f001..0c2d9553213 100644..100755 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -20,11 +20,16 @@ else exit 1 fi +if [ -z "$VESPA_LLVM_VERSION" ]; then + VESPA_LLVM_VERSION=5.0 +fi + cmake3 \ -DCMAKE_INSTALL_PREFIX=/opt/vespa \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ - -DEXTRA_LINK_DIRECTORY="/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ - -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-gtest/include;/opt/vespa-boost/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ - -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm3.9/lib" \ + -DEXTRA_LINK_DIRECTORY="/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ + -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-gtest/include;/opt/vespa-boost/include;/opt/vespa-cppunit/include;/usr/include/llvm$VESPA_LLVM_VERSION" \ + -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-gtest/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ ${EXTRA_CMAKE_ARGS} \ + -DVESPA_LLVM_VERSION=$VESPA_LLVM_VERSION \ "${SOURCE_DIR}" diff --git a/build_settings.cmake b/build_settings.cmake index a154559d1e0..1bb5ed8baa0 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -74,7 +74,7 @@ endif() if(VESPA_LLVM_VERSION) else() -set (VESPA_LLVM_VERSION "3.9") +set (VESPA_LLVM_VERSION "6.0") endif() if(VESPA_USER) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index d8c1276f7e8..6ace7a61f28 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -283,14 +283,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye */ public boolean delete(ApplicationId applicationId) { List<String> hostedZonesToUseDeleteApplication = - Arrays.asList("dev.us-east-1", "dev.corp-us-east-1", "test.us-east-1", - "prod.corp-us-east-1", "prod.aws-us-east-1a", "prod.aws-us-west-1b"); - boolean useDeleteApplicationLegacyInThisZone = !hostedZonesToUseDeleteApplication.contains(zone().toString()); + Arrays.asList("dev.us-east-1", "dev.corp-us-east-1", + "test.us-east-1", "staging.us-east-3", + "prod.aws-us-east-1a", "prod.aws-us-west-1b", + "prod.corp-us-east-1", "prod.us-central-1"); + boolean useDeleteApplicationLegacy = !hostedZonesToUseDeleteApplication.contains(zone().toString()); // TODO: Use deleteApplication() in all zones if (configserverConfig.deleteApplicationLegacy() || - (configserverConfig.hostedVespa() && zone().system() == SystemName.main && - useDeleteApplicationLegacyInThisZone)) { + (configserverConfig.hostedVespa() && zone().system() == SystemName.main && useDeleteApplicationLegacy)) { return deleteApplicationLegacy(applicationId); } else { return deleteApplication(applicationId); @@ -554,12 +555,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye listApplications().forEach(app -> tenantRepository.getTenant(app.tenant()).getLocalSessionRepo().purgeOldSessions()); } - public int deleteExpiredRemoteSessions(Duration expiryTime) { + public int deleteExpiredRemoteSessions(Duration expiryTime, boolean deleteFromZooKeeper ) { return listApplications() .stream() .map(app -> tenantRepository.getTenant(app.tenant()).getRemoteSessionRepo() - // TODO: Delete in all zones - .deleteExpiredSessions(expiryTime, zone().system() == SystemName.cd)) + .deleteExpiredSessions(expiryTime, deleteFromZooKeeper)) .mapToInt(i -> i) .sum(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index a0cbf4e4845..0631fca6a32 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; @@ -28,8 +30,14 @@ public class SessionsMaintainer extends Maintainer { // Expired remote sessions are not expected to exist, they should have been deleted when // a deployment happened or when the application was deleted. We still see them from time to time, // probably due to some race or another bug - Duration expiryTime = Duration.ofDays(30); - if (hostedVespa) - applicationRepository.deleteExpiredRemoteSessions(expiryTime); + if (hostedVespa) { + Duration expiryTime = Duration.ofDays(30); + Zone zone = applicationRepository.zone(); + // TODO: Delete in all zones + boolean deleteFromZooKeeper = zone.system() == SystemName.cd || + zone.environment().isTest() || + zone.region().value().equals("us-central-1"); + applicationRepository.deleteExpiredRemoteSessions(expiryTime, deleteFromZooKeeper); + } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index dbccba395a2..b91627374e6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -135,6 +135,7 @@ public class RemoteSession extends Session { public void delete() { Transaction transaction = zooKeeperClient.deleteTransaction(); transaction.commit(); + transaction.close(); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d9a653a1dc2..4a279c68c70 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -273,7 +273,7 @@ public class ApplicationRepositoryTest { assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId()); // There should be no expired remote sessions in the common case - assertEquals(0, applicationRepository.deleteExpiredRemoteSessions(Duration.ofSeconds(0))); + assertEquals(0, applicationRepository.deleteExpiredRemoteSessions(Duration.ofSeconds(0), true)); } private PrepareResult prepareAndActivateApp(File application) throws IOException { diff --git a/dist/vespa.spec b/dist/vespa.spec index bd2bf798e58..1a0a51b88fc 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -32,31 +32,26 @@ BuildRequires: libatomic BuildRequires: Judy-devel %if 0%{?centos} BuildRequires: cmake3 -BuildRequires: llvm3.9-devel +BuildRequires: llvm5.0-devel BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-gtest >= 1.8.0-1 %endif %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 BuildRequires: maven -%if 0%{?fc26} -BuildRequires: llvm-devel >= 4.0 -BuildRequires: boost-devel >= 1.63 -BuildRequires: vespa-gtest >= 1.8.0-2 -%endif %if 0%{?fc27} -BuildRequires: llvm4.0-devel >= 4.0 +BuildRequires: llvm-devel >= 5.0.2 BuildRequires: boost-devel >= 1.64 BuildRequires: vespa-gtest >= 1.8.0-2 %endif %if 0%{?fc28} -BuildRequires: llvm4.0-devel >= 4.0 +BuildRequires: llvm-devel >= 6.0.1 BuildRequires: boost-devel >= 1.66 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc29} -BuildRequires: llvm3.9-devel >= 3.9.1 +BuildRequires: llvm-devel >= 6.0.1 BuildRequires: boost-devel >= 1.66 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -104,36 +99,25 @@ Requires: perf Requires: gdb Requires: net-tools %if 0%{?centos} -Requires: llvm3.9 -%define _extra_link_directory /usr/lib64/llvm3.9/lib;/opt/vespa-gtest/lib;/opt/vespa-cppunit/lib -%define _extra_include_directory /usr/include/llvm3.9;/opt/vespa-boost/include;/opt/vespa-gtest/include;/opt/vespa-cppunit/include +Requires: llvm5.0 +%define _vespa_llvm_version 5.0 +%define _extra_link_directory /usr/lib64/llvm5.0/lib;/opt/vespa-gtest/lib;/opt/vespa-cppunit/lib +%define _extra_include_directory /usr/include/llvm5.0;/opt/vespa-boost/include;/opt/vespa-gtest/include;/opt/vespa-cppunit/include %endif %if 0%{?fedora} -%if 0%{?fc26} -Requires: llvm-libs >= 4.0 -%define _vespa_llvm_version 4.0 -%define _vespa_gtest_link_directory /opt/vespa-gtest/lib -%define _vespa_gtest_include_directory /opt/vespa-gtest/include -%endif %if 0%{?fc27} -Requires: llvm4.0-libs >= 4.0 -%define _vespa_llvm_version 4.0 -%define _vespa_llvm_link_directory /usr/lib64/llvm4.0/lib -%define _vespa_llvm_include_directory /usr/include/llvm4.0 +Requires: llvm-libs >= 5.0.2 +%define _vespa_llvm_version 5.0 %define _vespa_gtest_link_directory /opt/vespa-gtest/lib %define _vespa_gtest_include_directory /opt/vespa-gtest/include %endif %if 0%{?fc28} -Requires: llvm4.0-libs >= 4.0 -%define _vespa_llvm_version 4.0 -%define _vespa_llvm_link_directory /usr/lib64/llvm4.0/lib -%define _vespa_llvm_include_directory /usr/include/llvm4.0 +Requires: llvm-libs >= 6.0.1 +%define _vespa_llvm_version 6.0 %endif %if 0%{?fc29} -Requires: llvm3.9-libs >= 3.9.1 -%define _vespa_llvm_version 3.9 -%define _vespa_llvm_link_directory /usr/lib64/llvm3.9/lib -%define _vespa_llvm_include_directory /usr/include/llvm3.9 +Requires: llvm-libs >= 6.0.1 +%define _vespa_llvm_version 6.0 %endif %define _extra_link_directory /opt/vespa-cppunit/lib%{?_vespa_llvm_link_directory:;%{_vespa_llvm_link_directory}}%{?_vespa_gtest_link_directory:;%{_vespa_gtest_link_directory}} %define _extra_include_directory /opt/vespa-cppunit/include%{?_vespa_llvm_include_directory:;%{_vespa_llvm_include_directory}}%{?_vespa_gtest_include_directory:;%{_vespa_gtest_include_directory}} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index a8403b8b10d..f82047d885c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -1,8 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.identity; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; @@ -12,7 +10,6 @@ import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.client.InstanceCsrGenerator; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; @@ -53,9 +50,9 @@ public class AthenzCredentialsMaintainer { private static final Duration EXPIRY_MARGIN = Duration.ofDays(1); private static final Duration REFRESH_PERIOD = Duration.ofDays(1); - private static final Path CONTAINER_SIA_DIRECTORY = Paths.get("/var/lib/sia"); + private static final Duration REFRESH_BACKOFF = Duration.ofHours(1); // Backoff when refresh fails to ensure ZTS is not DDoS'ed. - private static final ObjectMapper mapper = new ObjectMapper().registerModule(new JavaTimeModule()); + private static final Path CONTAINER_SIA_DIRECTORY = Paths.get("/var/lib/sia"); private final boolean enabled; private final PrefixLogger log; @@ -72,6 +69,8 @@ public class AthenzCredentialsMaintainer { private final InstanceCsrGenerator csrGenerator; private final AthenzService configserverIdentity; + private Instant lastRefreshAttempt = Instant.EPOCH; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts + public AthenzCredentialsMaintainer(String hostname, Environment environment, ServiceIdentityProvider hostIdentityProvider) { @@ -97,14 +96,11 @@ public class AthenzCredentialsMaintainer { this.clock = Clock.systemUTC(); } - /** - * @return Returns true if credentials were updated - */ - public boolean converge() { + public void converge() { try { if (!enabled) { log.debug("Feature disabled on this host - not fetching certificate"); - return false; + return; } log.debug("Checking certificate"); Instant now = clock.instant(); @@ -114,23 +110,29 @@ public class AthenzCredentialsMaintainer { Files.createDirectories(certificateFile.getParent()); Files.createDirectories(identityDocumentFile.getParent()); registerIdentity(); - return true; + return; } X509Certificate certificate = readCertificateFromFile(); Instant expiry = certificate.getNotAfter().toInstant(); if (isCertificateExpired(expiry, now)) { log.info(String.format("Certificate has expired (expiry=%s)", expiry.toString())); registerIdentity(); - return true; + return; } Duration age = Duration.between(certificate.getNotBefore().toInstant(), now); if (shouldRefreshCredentials(age)) { log.info(String.format("Certificate is ready to be refreshed (age=%s)", age.toString())); - refreshIdentity(); - return true; + if (shouldThrottleRefreshAttempts(now)) { + log.warning(String.format("Skipping refresh attempt as last refresh was on %s (less than %s ago)", + lastRefreshAttempt.toString(), REFRESH_BACKOFF.toString())); + return; + } else { + lastRefreshAttempt = now; + refreshIdentity(); + return; + } } log.debug("Certificate is still valid"); - return false; } catch (IOException e) { throw new UncheckedIOException(e); } @@ -154,6 +156,10 @@ public class AthenzCredentialsMaintainer { return age.compareTo(REFRESH_PERIOD) >= 0; } + private boolean shouldThrottleRefreshAttempts(Instant now) { + return REFRESH_BACKOFF.compareTo(Duration.between(lastRefreshAttempt, now)) > 0; + } + private X509Certificate readCertificateFromFile() throws IOException { String pemEncodedCertificate = new String(Files.readAllBytes(certificateFile)); return X509CertificateUtils.fromPem(pemEncodedCertificate); @@ -177,7 +183,7 @@ public class AthenzCredentialsMaintainer { EntityBindingsMapper.toAttestationData(signedIdentityDocument), false, csr); - writeIdentityDocument(signedIdentityDocument); + EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, signedIdentityDocument); writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); log.info("Instance successfully registered and credentials written to file"); } catch (IOException e) { @@ -186,7 +192,7 @@ public class AthenzCredentialsMaintainer { } private void refreshIdentity() { - SignedIdentityDocument identityDocument = readIdentityDocument(); + SignedIdentityDocument identityDocument = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); Pkcs10Csr csr = csrGenerator.generateCsr(containerIdentity, identityDocument.providerUniqueId(), identityDocument.ipAddresses(), keyPair); SSLContext containerIdentitySslContext = @@ -194,45 +200,27 @@ public class AthenzCredentialsMaintainer { .withKeyStore(privateKeyFile.toFile(), certificateFile.toFile()) .withTrustStore(trustStorePath.toFile(), KeyStoreType.JKS) .build(); - try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, containerIdentity, containerIdentitySslContext)) { - InstanceIdentity instanceIdentity = - ztsClient.refreshInstance( - configserverIdentity, - containerIdentity, - identityDocument.providerUniqueId().asDottedString(), - false, - csr); - writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); - log.info("Instance successfully refreshed and credentials written to file"); - } catch (ZtsClientException e) { - // TODO Find out why certificate was revoked and hopefully remove this workaround - if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { - log.error("Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); - registerIdentity(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private SignedIdentityDocument readIdentityDocument() { try { - SignedIdentityDocumentEntity entity = mapper.readValue(identityDocumentFile.toFile(), SignedIdentityDocumentEntity.class); - return EntityBindingsMapper.toSignedIdentityDocument(entity); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void writeIdentityDocument(SignedIdentityDocument signedIdentityDocument) { - try { - SignedIdentityDocumentEntity entity = - EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument); - Path tempIdentityDocumentFile = toTempPath(identityDocumentFile); - mapper.writeValue(tempIdentityDocumentFile.toFile(), entity); - Files.move(tempIdentityDocumentFile, identityDocumentFile, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - throw new UncheckedIOException(e); + try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, containerIdentity, containerIdentitySslContext)) { + InstanceIdentity instanceIdentity = + ztsClient.refreshInstance( + configserverIdentity, + containerIdentity, + identityDocument.providerUniqueId().asDottedString(), + false, + csr); + writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); + log.info("Instance successfully refreshed and credentials written to file"); + } catch (ZtsClientException e) { + if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { + log.error("Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); + registerIdentity(); + } else { + throw e; + } + } + } catch (Exception e) { + log.error("Certificate refresh failed: " + e.getMessage(), e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 5db79beac3c..69b31f506e5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; +import com.yahoo.vespa.hosted.provision.provisioning.OsVersions; import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; import java.time.Clock; @@ -78,6 +79,7 @@ public class NodeRepository extends AbstractComponent { private final NodeFlavors flavors; private final NameResolver nameResolver; private final DockerImage dockerImage; + private final OsVersions osVersions; /** * Creates a node repository from a zookeeper provider. @@ -100,6 +102,7 @@ public class NodeRepository extends AbstractComponent { this.flavors = flavors; this.nameResolver = nameResolver; this.dockerImage = dockerImage; + this.osVersions = new OsVersions(this.db); // read and write all nodes to make sure they are stored in the latest version of the serialized format for (Node.State state : Node.State.values()) @@ -115,6 +118,9 @@ public class NodeRepository extends AbstractComponent { /** @return The name resolver used to resolve hostname and ip addresses */ public NameResolver nameResolver() { return nameResolver; } + /** Returns the OS versions to use for nodes in this */ + public OsVersions osVersions() { return osVersions; } + // ---------------- Query API ---------------------------------------------------------------- /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 19e34ccb169..feaa4d8241d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -22,6 +22,7 @@ public class Status { private final boolean wantToRetire; private final boolean wantToDeprovision; private final Optional<String> hardwareDivergence; + private final Optional<Version> osVersion; public Status(Generation generation, Optional<Version> vespaVersion, @@ -29,50 +30,49 @@ public class Status { Optional<String> hardwareFailureDescription, boolean wantToRetire, boolean wantToDeprovision, - Optional<String> hardwareDivergence) { - Objects.requireNonNull(generation, "Generation must be non-null"); - Objects.requireNonNull(vespaVersion, "Vespa version must be non-null"); - Objects.requireNonNull(hardwareFailureDescription, "Hardware failure description must be non-null"); + Optional<String> hardwareDivergence, + Optional<Version> osVersion) { Objects.requireNonNull(hardwareDivergence, "Hardware divergence must be non-null"); hardwareDivergence.ifPresent(s -> requireNonEmptyString(s, "Hardware divergence must be non-empty")); - this.reboot = generation; - this.vespaVersion = vespaVersion; + this.reboot = Objects.requireNonNull(generation, "Generation must be non-null"); + this.vespaVersion = Objects.requireNonNull(vespaVersion, "Vespa version must be non-null"); this.failCount = failCount; - this.hardwareFailureDescription = hardwareFailureDescription; + this.hardwareFailureDescription = Objects.requireNonNull(hardwareFailureDescription, "Hardware failure description must be non-null"); this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; this.hardwareDivergence = hardwareDivergence; + this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null"); } /** Returns a copy of this with the reboot generation changed */ - public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns the reboot generation of this node */ public Generation reboot() { return reboot; } /** Returns a copy of this with the vespa version changed */ - public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns the Vespa version installed on the node, if known */ public Optional<Version> vespaVersion() { return vespaVersion; } - public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, failCount + 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } - public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, failCount - 1, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } - public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, value, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } - public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); } + public Status withHardwareFailureDescription(Optional<String> hardwareFailureDescription) { return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns the type of the last hardware failure detected on this node, or empty if none */ public Optional<String> hardwareFailureDescription() { return hardwareFailureDescription; } /** Returns a copy of this with the want to retire flag changed */ public Status withWantToRetire(boolean wantToRetire) { - return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** @@ -85,7 +85,7 @@ public class Status { /** Returns a copy of this with the want to de-provision flag changed */ public Status withWantToDeprovision(boolean wantToDeprovision) { - return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** @@ -96,14 +96,27 @@ public class Status { } public Status withHardwareDivergence(Optional<String> hardwareDivergence) { - return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence); + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, osVersion); } /** Returns hardware divergence report as JSON string, if any */ public Optional<String> hardwareDivergence() { return hardwareDivergence; } + /** Returns a copy of this with the current OS version set to version */ + public Status withOsVersion(Version version) { + return new Status(reboot, vespaVersion, failCount, hardwareFailureDescription, wantToRetire, wantToDeprovision, hardwareDivergence, Optional.of(version)); + } + + /** Returns the current OS version of this node, if any */ + public Optional<Version> osVersion() { + return osVersion; + } + /** Returns the initial status of a newly provisioned node */ - public static Status initial() { return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false, false, Optional.empty()); } + public static Status initial() { + return new Status(Generation.inital(), Optional.empty(), 0, Optional.empty(), false, + false, Optional.empty(), Optional.empty()); + } private void requireNonEmptyString(String value, String message) { Objects.requireNonNull(value, message); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java new file mode 100644 index 00000000000..f7083a6398f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeOsVersionFilter.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.Objects; + +/** + * Filter nodes by their OS version. + * + * @author mpolden + */ +public class NodeOsVersionFilter extends NodeFilter { + + private final Version version; + + private NodeOsVersionFilter(Version version, NodeFilter next) { + super(next); + this.version = Objects.requireNonNull(version, "version cannot be null"); + } + + @Override + public boolean matches(Node node) { + if (!version.isEmpty() && !node.status().osVersion().filter(v -> v.equals(version)).isPresent()) { + return false; + } + return nextMatches(node); + } + + public static NodeOsVersionFilter from(String version, NodeFilter filter) { + return new NodeOsVersionFilter(Version.fromString(version), filter); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index f559ec0037b..a5dfc616302 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -71,6 +71,7 @@ public class CuratorDatabaseClient { curatorDatabase.create(toPath(state)); curatorDatabase.create(inactiveJobsPath()); curatorDatabase.create(infrastructureVersionsPath()); + curatorDatabase.create(osVersionsPath()); } /** @@ -374,4 +375,23 @@ public class CuratorDatabaseClient { return root.append("infrastructureVersions"); } + public Map<NodeType, Version> readOsVersions() { + return read(osVersionsPath(), NodeTypeVersionsSerializer::fromJson).orElseGet(TreeMap::new); + } + + public void writeOsVersions(Map<NodeType, Version> versions) { + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.setData(osVersionsPath().getAbsolute(), + NodeTypeVersionsSerializer.toJson(versions))); + transaction.commit(); + } + + public Lock lockOsVersions() { + return lock(lockRoot.append("osVersionsLock"), defaultLockTimeout); + } + + private Path osVersionsPath() { + return root.append("osVersions"); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 669f2063ee6..dbe6589dd7f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -58,6 +58,7 @@ public class NodeSerializer { private static final String wantToRetireKey = "wantToRetire"; private static final String wantToDeprovisionKey = "wantToDeprovision"; private static final String hardwareDivergenceKey = "hardwareDivergence"; + private static final String osVersionKey = "osVersion"; // Configuration fields private static final String flavorKey = "flavor"; @@ -114,6 +115,7 @@ public class NodeSerializer { object.setString(nodeTypeKey, toString(node.type())); node.status().hardwareDivergence().ifPresent(hardwareDivergence -> object.setString(hardwareDivergenceKey, hardwareDivergence)); + node.status().osVersion().ifPresent(version -> object.setString(osVersionKey, version.toString())); } private void toSlime(Allocation allocation, Cursor object) { @@ -169,7 +171,8 @@ public class NodeSerializer { hardwareFailureDescriptionFromSlime(object), object.field(wantToRetireKey).asBool(), object.field(wantToDeprovisionKey).asBool(), - removeQuotedNulls(hardwareDivergenceFromSlime(object))); + removeQuotedNulls(hardwareDivergenceFromSlime(object)), + versionFromSlime(object.field(osVersionKey))); } private Flavor flavorFromSlime(Inspector object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java new file mode 100644 index 00000000000..7e941d58a62 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersions.java @@ -0,0 +1,91 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Thread-safe class that manages target OS versions for nodes in this repository. + * + * The target OS version for each node type is set through the /nodes/v2/upgrade REST API. + * + * @author mpolden + */ +public class OsVersions { + + private static final Duration defaultCacheTtl = Duration.ofMinutes(1); + private static final Logger log = Logger.getLogger(OsVersions.class.getName()); + + private final CuratorDatabaseClient db; + private final Duration cacheTtl; + + /** + * Target OS version is read on every request to /nodes/v2/node/[fqdn]. Cache current targets to avoid + * unnecessary ZK reads. When targets change, some nodes may need to wait for TTL until they see the new target, + * this is fine. + */ + private volatile Supplier<Map<NodeType, Version>> currentTargets; + + public OsVersions(CuratorDatabaseClient db) { + this(db, defaultCacheTtl); + } + + OsVersions(CuratorDatabaseClient db, Duration cacheTtl) { + this.db = db; + this.cacheTtl = cacheTtl; + createCache(); + } + + private void createCache() { + this.currentTargets = Suppliers.memoizeWithExpiration(() -> ImmutableMap.copyOf(db.readOsVersions()), + cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Returns the current target versions for each node type */ + public Map<NodeType, Version> targets() { + return currentTargets.get(); + } + + /** Returns the current target version for given node type, if any */ + public Optional<Version> targetFor(NodeType type) { + return Optional.ofNullable(targets().get(type)); + } + + /** Set the target OS version for nodes of given type */ + public void setTarget(NodeType nodeType, Version newTarget, boolean force) { + if (!nodeType.isDockerHost()) { + throw new IllegalArgumentException("Setting target OS version for " + nodeType + " nodes is unsupported"); + } + try (Lock lock = db.lockOsVersions()) { + Map<NodeType, Version> osVersions = db.readOsVersions(); + Optional<Version> oldTarget = Optional.ofNullable(osVersions.get(nodeType)); + + if (oldTarget.filter(v -> v.equals(newTarget)).isPresent()) { + return; // Old target matches new target, nothing to do + } + + if (!force && oldTarget.filter(v -> v.isAfter(newTarget)).isPresent()) { + throw new IllegalArgumentException("Cannot set target OS version to " + newTarget + + " without setting 'force', as it's lower than the current version: " + + oldTarget.get()); + } + + osVersions.put(nodeType, newTarget); + db.writeOsVersions(osVersions); + createCache(); // Throw away current cache + log.info("Set OS target version for " + nodeType + " nodes to " + newTarget.toFullString()); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 31d9a606d91..910da4e90bf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -114,6 +114,8 @@ public class NodePatcher { case "vespaVersion" : case "currentVespaVersion" : return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); + case "currentOsVersion" : + return node.with(node.status().withOsVersion(Version.fromString(asString(value)))); case "failCount" : return node.with(node.status().setFailCount(asLong(value).intValue())); case "flavor" : diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 54202a15971..c282993a466 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.component.Version; import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -15,12 +16,12 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; @@ -28,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; +import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -37,7 +39,6 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; -import javax.inject.Inject; import static com.yahoo.vespa.config.SlimeUtils.optionalString; @@ -53,7 +54,6 @@ public class NodesApiHandler extends LoggingRequestHandler { private final NodeRepository nodeRepository; private final NodeRepositoryMaintenance maintenance; private final NodeFlavors nodeFlavors; - private static final String nodeTypeKey = "type"; @Inject public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, @@ -100,7 +100,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl()); - if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions()); + if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions()); throw new NotFoundException("Nothing at path '" + path + "'"); } @@ -114,18 +114,15 @@ public class NodesApiHandler extends LoggingRequestHandler { } else if (path.startsWith("/nodes/v2/state/failed/")) { List<Node> failedNodes = nodeRepository.failRecursively(lastElement(path), Agent.operator, "Failed through the nodes/v2 API"); - String failedHostnames = failedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - return new MessageResponse("Moved " + failedHostnames + " to failed"); + return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to failed"); } else if (path.startsWith("/nodes/v2/state/parked/")) { List<Node> parkedNodes = nodeRepository.parkRecursively(lastElement(path), Agent.operator, "Parked through the nodes/v2 API"); - String parkedHostnames = parkedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - return new MessageResponse("Moved " + parkedHostnames + " to parked"); + return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked"); } else if (path.startsWith("/nodes/v2/state/dirty/")) { List<Node> dirtiedNodes = nodeRepository.dirtyRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API"); - String dirtiedHostnames = dirtiedNodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - return new MessageResponse("Moved " + dirtiedHostnames + " to dirty"); + return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty"); } else if (path.startsWith("/nodes/v2/state/active/")) { nodeRepository.reactivate(lastElement(path), Agent.operator, "Reactivated through nodes/v2 API"); @@ -143,7 +140,7 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse("Updated " + node.hostname()); } else if (path.startsWith("/nodes/v2/upgrade/")) { - return setInfrastructureVersion(request); + return setTargetVersions(request); } throw new NotFoundException("Nothing at '" + path + "'"); @@ -227,10 +224,10 @@ public class NodesApiHandler extends LoggingRequestHandler { additionalIpAddresses, parentHostname, nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString()), - nodeTypeFromSlime(inspector.field(nodeTypeKey))); + nodeTypeFromSlime(inspector.field("type"))); } - private NodeType nodeTypeFromSlime(Inspector object) { + private static NodeType nodeTypeFromSlime(Inspector object) { if (! object.valid()) return NodeType.tenant; // default switch (object.asString()) { case "tenant" : return NodeType.tenant; @@ -252,18 +249,19 @@ public class NodesApiHandler extends LoggingRequestHandler { filter = StateFilter.from(request.getProperty("state"), filter); filter = NodeTypeFilter.from(request.getProperty("type"), filter); filter = ParentHostFilter.from(request.getProperty("parentHost"), filter); + filter = NodeOsVersionFilter.from(request.getProperty("osVersion"), filter); return filter; } - private String lastElement(String path) { + private static String lastElement(String path) { if (path.endsWith("/")) path = path.substring(0, path.length()-1); int lastSlash = path.lastIndexOf("/"); if (lastSlash < 0) return path; - return path.substring(lastSlash + 1, path.length()); + return path.substring(lastSlash + 1); } - private boolean isPatchOverride(HttpRequest request) { + private static boolean isPatchOverride(HttpRequest request) { // Since Jersey's HttpUrlConnector does not support PATCH we support this by override this on POST requests. String override = request.getHeader("X-HTTP-Method-Override"); if (override != null) { @@ -284,18 +282,37 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); } - private MessageResponse setInfrastructureVersion(HttpRequest request) { + private MessageResponse setTargetVersions(HttpRequest request) { NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); Inspector inspector = toSlime(request.getData()).get(); + List<String> messageParts = new ArrayList<>(2); - Inspector versionField = inspector.field("version"); - if (!versionField.valid()) - throw new IllegalArgumentException("'version' is missing"); - Version version = Version.fromString(versionField.asString()); boolean force = inspector.field("force").asBool(); + Inspector versionField = inspector.field("version"); + Inspector osVersionField = inspector.field("osVersion"); + + if (versionField.valid()) { + Version version = Version.fromString(versionField.asString()); + maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force); + messageParts.add("version to " + version.toFullString()); + } + + if (osVersionField.valid()) { + Version osVersion = Version.fromString(osVersionField.asString()); + nodeRepository.osVersions().setTarget(nodeType, osVersion, force); + messageParts.add("osVersion to " + osVersion.toFullString()); + } - maintenance.infrastructureVersions().setTargetVersion(nodeType, version, force); + if (messageParts.isEmpty()) { + throw new IllegalArgumentException("At least one of 'version' and 'osVersion' must be set"); + } - return new MessageResponse("Set version for " + nodeType + " to " + version.toFullString()); + return new MessageResponse("Set " + String.join(", ", messageParts) + + " for nodes of type " + nodeType); } + + private static String hostnamesAsString(List<Node> nodes) { + return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 99ebb3e517b..970871a4d05 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -169,6 +169,8 @@ class NodesResponse extends HttpResponse { } object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); + node.status().osVersion().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); + nodeRepository.osVersions().targetFor(node.type()).ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); node.status().vespaVersion() .filter(version -> !version.isEmpty()) .ifPresent(version -> { @@ -231,7 +233,7 @@ class NodesResponse extends HttpResponse { path = path.substring(0, path.length()-1); int lastSlash = path.lastIndexOf("/"); if (lastSlash < 0) return path; - return path.substring(lastSlash+1, path.length()); + return path.substring(lastSlash+1); } private static Node.State stateFromString(String stateString) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java index 3fb712e182f..392cba7baa9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java @@ -6,24 +6,25 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; +import com.yahoo.vespa.hosted.provision.provisioning.OsVersions; import java.io.IOException; import java.io.OutputStream; -import java.util.Comparator; -import java.util.Map; /** - * A response containing infrastructure versions + * A response containing targets for infrastructure Vespa version and OS version. * * @author freva */ public class UpgradeResponse extends HttpResponse { private final InfrastructureVersions infrastructureVersions; + private final OsVersions osVersions; - public UpgradeResponse(InfrastructureVersions infrastructureVersions) { + public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions) { super(200); this.infrastructureVersions = infrastructureVersions; + this.osVersions = osVersions; } @Override @@ -34,6 +35,9 @@ public class UpgradeResponse extends HttpResponse { Cursor versionsObject = root.setObject("versions"); infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString())); + Cursor osVersionsObject = root.setObject("osVersions"); + osVersions.targets().forEach((nodeType, version) -> osVersionsObject.setString(nodeType.name(), version.toFullString())); + new JsonFormat(true).encode(stream, slime); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java new file mode 100644 index 00000000000..4639a86aeec --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializerTest.java @@ -0,0 +1,28 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import org.junit.Test; + +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class NodeTypeVersionsSerializerTest { + + @Test + public void test_serialization() { + Map<NodeType, Version> versions = new TreeMap<>(); + versions.put(NodeType.host, Version.fromString("7.1")); + versions.put(NodeType.confighost, Version.fromString("7.2")); + + Map<NodeType, Version> serialized = NodeTypeVersionsSerializer.fromJson(NodeTypeVersionsSerializer.toJson(versions)); + assertEquals(versions, serialized); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 0c32c13f387..6c9d0be69b2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import org.junit.Test; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Arrays; @@ -240,7 +239,7 @@ public class SerializationTest { } @Test - public void serialize_additional_ip_addresses() throws IOException { + public void serialize_additional_ip_addresses() { Node node = createNode(); // Test round-trip with additional addresses @@ -326,7 +325,7 @@ public class SerializationTest { } @Test - public void vespa_version_serialization() throws Exception { + public void vespa_version_serialization() { String nodeWithWantedVespaVersion = "{\n" + " \"type\" : \"tenant\",\n" + @@ -343,6 +342,18 @@ public class SerializationTest { assertEquals("6.42.2", node.allocation().get().membership().cluster().vespaVersion().toString()); } + @Test + public void os_version_serialization() { + Node serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(createNode())); + assertFalse(serialized.status().osVersion().isPresent()); + + // Update OS version + serialized = serialized.with(serialized.status() + .withOsVersion(Version.fromString("7.1"))); + serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(serialized)); + assertEquals(Version.fromString("7.1"), serialized.status().osVersion().get()); + } + private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; if (ipAddress.length > 0) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java new file mode 100644 index 00000000000..88f5dcb9854 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/OsVersionsTest.java @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author mpolden + */ +public class OsVersionsTest { + + private OsVersions versions; + + @Before + public void before() { + versions = new OsVersions( + new NodeRepositoryTester().nodeRepository().database(), + Duration.ofDays(1) // Long TTL to avoid timed expiry during test + ); + } + + @Test + public void test_versions() { + assertTrue("No versions set", versions.targets().isEmpty()); + assertSame("Caches empty target versions", versions.targets(), versions.targets()); + + // Upgrade OS + Version version1 = Version.fromString("7.1"); + versions.setTarget(NodeType.host, version1, false); + Map<NodeType, Version> targetVersions = versions.targets(); + assertSame("Caches target versions", targetVersions, versions.targets()); + assertEquals(version1, versions.targetFor(NodeType.host).get()); + + // Upgrade OS again + Version version2 = Version.fromString("7.2"); + versions.setTarget(NodeType.host, version2, false); + assertNotSame("Cache invalidated", targetVersions, versions.targets()); + assertEquals(version2, versions.targetFor(NodeType.host).get()); + + // Downgrading fails + try { + versions.setTarget(NodeType.host, version1, false); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + // Forcing downgrade succeeds + versions.setTarget(NodeType.host, version1, true); + assertEquals(version1, versions.targetFor(NodeType.host).get()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index ef0feecc037..448b64d1e78 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -506,21 +506,21 @@ public class RestApiTest { @Test public void test_upgrade() throws IOException { // Initially, no versions are set - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{}}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{},\"osVersions\":{}}"); // Set version for config and confighost assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", Utf8.toBytes("{\"version\": \"6.123.456\"}"), Request.Method.PATCH), - "{\"message\":\"Set version for config to 6.123.456\"}"); + "{\"message\":\"Set version to 6.123.456 for nodes of type config\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{\"version\": \"6.123.456\"}"), Request.Method.PATCH), - "{\"message\":\"Set version for confighost to 6.123.456\"}"); + "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}"); // Verify versions are set assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\"},\"osVersions\":{}}"); // Downgrade without force fails assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -534,11 +534,84 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"), Request.Method.PATCH), - "{\"message\":\"Set version for confighost to 6.123.1\"}"); + "{\"message\":\"Set version to 6.123.1 for nodes of type confighost\"}"); // Verify version has been updated assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"}}"); + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{}}"); + + // Upgrade OS for confighost and host + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type confighost\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); + + // OS versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"}}"); + + // Upgrade OS and Vespa together + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.124.42\", \"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}"); + + // Attempt to upgrade unsupported node type + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting target OS version for config nodes is unsupported\"}"); + + // Attempt to downgrade OS + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); + + // Downgrading OS with force succeeds + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\", \"force\": true}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}"); + } + + @Test + public void test_os_version() throws Exception { + // Schedule OS upgrade + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); + + // Other node type does not return wanted OS version + Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); + assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); + + // Node updates its node object after upgrading OS + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); + + // Another node upgrades + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost2.yahoo.com\"}"); + + // Filter nodes by osVersion + assertResponse(new Request("http://localhost:8080/nodes/v2/node/?osVersion=7.5.2"), + "{\"nodes\":[" + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" + + "]}"); } /** Tests the rendering of each node separately to make it easier to find errors */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json new file mode 100644 index 00000000000..2e8092012fb --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json @@ -0,0 +1,72 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "currentOsVersion": "7.5.2", + "wantedOsVersion": "7.5.2", + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json new file mode 100644 index 00000000000..88bda7544d9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json @@ -0,0 +1,71 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "wantedOsVersion": "7.5.2", + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 3bfe492a125..641c3001795 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -11,7 +11,10 @@ import com.yahoo.vespa.athenz.identityprovider.api.bindings.VespaUniqueInstanceI import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.Base64; import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId.fromDottedString; @@ -116,7 +119,9 @@ public class EntityBindingsMapper { public static void writeSignedIdentityDocumentToFile(Path file, SignedIdentityDocument document) { try { SignedIdentityDocumentEntity entity = EntityBindingsMapper.toSignedIdentityDocumentEntity(document); - mapper.writeValue(file.toFile(), entity); + Path tempFile = Paths.get(file.toAbsolutePath().toString() + ".tmp"); + mapper.writeValue(tempFile.toFile(), entity); + Files.move(tempFile, file, StandardCopyOption.ATOMIC_MOVE); } catch (IOException e) { throw new UncheckedIOException(e); } |