diff options
253 files changed, 4146 insertions, 3317 deletions
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index bfd02d54d43..57339159e3d 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -14,67 +14,6 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> - <!-- COMPILE --> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zms-java-client</artifactId> - <scope>compile</scope> - <exclusions> - <!-- Provided by JDisc / container-dev --> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <!--Exclude all Jersey bundles provided by JDisc--> - <exclusion> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-json-jackson</artifactId> - </exclusion> - <!--Exclude all Jackson bundles provided by JDisc --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zts-java-client</artifactId> - <scope>compile</scope> - <exclusions> - <!-- Provided by JDisc / container-dev --> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <!--Exclude all Jackson bundles provided by JDisc --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- PROVIDED --> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -110,6 +49,30 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-athenz</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + <scope>compile</scope> + </dependency> <!-- TEST --> <dependency> @@ -130,18 +93,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.4.1</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <version>4.4.1</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentials.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzCredentials.java index 790a7c54333..36c1aee49e0 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentials.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzCredentials.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import java.security.KeyPair; import java.security.cert.X509Certificate; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentialsService.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzCredentialsService.java index 5786eb9e398..4072568d9d2 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentialsService.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzCredentialsService.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.core.identity.IdentityConfig; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzIdentityProviderImpl.java index c32d08c97ff..18f90ce545f 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzIdentityProviderImpl.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; @@ -8,21 +8,12 @@ import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.athenz.api.AthenzIdentityCertificate; +import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import java.io.FileInputStream; -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; +import java.io.File; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -62,9 +53,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen static final String METRICS_UPDATER_TAG = "metrics-updater"; - private final AtomicReference<AthenzCredentials> credentials = new AtomicReference<>(); + private volatile AthenzCredentials credentials; private final AtomicReference<Throwable> lastThrowable = new AtomicReference<>(); - private final CountDownLatch credentialsRetrievedSignal = new CountDownLatch(1); private final AthenzCredentialsService athenzCredentialsService; private final Scheduler scheduler; private final Clock clock; @@ -96,26 +86,17 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.clock = clock; this.domain = config.domain(); this.service = config.service(); - scheduler.submit(new RegisterInstanceTask()); - scheduler.schedule(new TimeoutInitialWaitTask(), INITIAL_WAIT_NTOKEN); - metricUpdater = new CertificateExpiryMetricUpdater(metric); + registerInstance(); } - @Override - public String getNToken() { + private void registerInstance() { try { - credentialsRetrievedSignal.await(); - AthenzCredentials credentialsSnapshot = credentials.get(); - if (credentialsSnapshot == null) { - throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", lastThrowable.get()); - } - if (isExpired(credentialsSnapshot)) { - throw new AthenzIdentityProviderException("Athenz credentials are expired", lastThrowable.get()); - } - return credentialsSnapshot.getNToken(); - } catch (InterruptedException e) { - throw new AthenzIdentityProviderException("Failed to register instance credentials", lastThrowable.get()); + credentials = athenzCredentialsService.registerInstance(); + scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD); + scheduler.submit(metricUpdater); + } catch (Throwable t) { + throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", t); } } @@ -130,49 +111,13 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } @Override - public SSLContext getSslContext() { - try { - SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); - sslContext.init(createKeyManagersWithServiceCertificate(), - createTrustManagersWithAthenzCa(), - null); - return sslContext; - } catch (NoSuchAlgorithmException | KeyManagementException e) { - throw new RuntimeException(e); - } - } - - private KeyManager[] createKeyManagersWithServiceCertificate() { - try { - credentialsRetrievedSignal.await(); - KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(null); - keyStore.setKeyEntry("instance-key", - credentials.get().getKeyPair().getPrivate(), - new char[0], - new Certificate[]{credentials.get().getCertificate()}); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, new char[0]); - return keyManagerFactory.getKeyManagers(); - } catch (KeyStoreException | NoSuchAlgorithmException | UnrecoverableKeyException | CertificateException | IOException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new AthenzIdentityProviderException("Failed to register instance credentials", lastThrowable.get()); - } - } - - private static TrustManager[] createTrustManagersWithAthenzCa() { - try { - KeyStore trustStore = KeyStore.getInstance("JKS"); - try (FileInputStream in = new FileInputStream("/opt/yahoo/share/ssl/certs/yahoo_certificate_bundle.jks")) { - trustStore.load(in, null); - } - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(trustStore); - return trustManagerFactory.getTrustManagers(); - } catch (CertificateException | IOException | KeyStoreException | NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + public SSLContext getIdentitySslContext() { + return new AthenzSslContextBuilder() + .withIdentityCertificate(new AthenzIdentityCertificate( + credentials.getCertificate(), + credentials.getKeyPair().getPrivate())) + .withTrustStore(new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")), "JKS") + .build(); } @Override @@ -188,56 +133,19 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen return credentials.getCreatedAt().plus(EXPIRES_AFTER).minus(EXPIRATION_MARGIN); } - private class RegisterInstanceTask implements RunnableWithTag { - - private final Duration backoffDelay; - - RegisterInstanceTask() { - this(INITIAL_BACKOFF_DELAY); - } - - RegisterInstanceTask(Duration backoffDelay) { - this.backoffDelay = backoffDelay; - } - - @Override - public void run() { - try { - credentials.set(athenzCredentialsService.registerInstance()); - credentialsRetrievedSignal.countDown(); - scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD); - scheduler.submit(metricUpdater); - } catch (Throwable t) { - log.log(LogLevel.ERROR, "Failed to register instance: " + t.getMessage(), t); - lastThrowable.set(t); - Duration nextBackoffDelay = backoffDelay.multipliedBy(BACKOFF_DELAY_MULTIPLIER); - if (nextBackoffDelay.compareTo(MAX_REGISTER_BACKOFF_DELAY) > 0) { - nextBackoffDelay = MAX_REGISTER_BACKOFF_DELAY; - } - scheduler.schedule(new RegisterInstanceTask(nextBackoffDelay), backoffDelay); - } - } - - @Override - public String tag() { - return REGISTER_INSTANCE_TAG; - } - } - private class UpdateCredentialsTask implements RunnableWithTag { @Override public void run() { - AthenzCredentials currentCredentials = credentials.get(); try { - AthenzCredentials newCredentials = isExpired(currentCredentials) + AthenzCredentials newCredentials = isExpired(credentials) ? athenzCredentialsService.registerInstance() - : athenzCredentialsService.updateCredentials(currentCredentials); - credentials.set(newCredentials); + : athenzCredentialsService.updateCredentials(credentials); + credentials = newCredentials; scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD); } catch (Throwable t) { log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); lastThrowable.set(t); - Duration timeToExpiration = Duration.between(clock.instant(), getExpirationTime(currentCredentials)); + Duration timeToExpiration = Duration.between(clock.instant(), getExpirationTime(credentials)); // NOTE: Update period might be after timeToExpiration, still we do not want to DDoS Athenz. Duration updatePeriod = timeToExpiration.compareTo(UPDATE_PERIOD) > 0 ? UPDATE_PERIOD : REDUCED_UPDATE_PERIOD; @@ -260,7 +168,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen @Override public void run() { - Instant expirationTime = getExpirationTime(credentials.get()); + Instant expirationTime = getExpirationTime(credentials); Duration remainingLifetime = Duration.between(clock.instant(), expirationTime); metric.set(CERTIFICATE_EXPIRY_METRIC_NAME, remainingLifetime.getSeconds(), null); scheduler.schedule(this, CERTIFICATE_EXPIRY_METRIC_UPDATE_PERIOD); @@ -272,18 +180,6 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } - private class TimeoutInitialWaitTask implements RunnableWithTag { - @Override - public void run() { - credentialsRetrievedSignal.countDown(); - } - - @Override - public String tag() { - return TIMEOUT_INITIAL_WAIT_TAG; - } - } - private static class ThreadPoolScheduler implements Scheduler { private static final Logger log = Logger.getLogger(ThreadPoolScheduler.class.getName()); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzService.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzService.java index 898f90e3438..c9e3809ea96 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzService.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzService.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtils.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/CryptoUtils.java index 388b40a1fe0..6a766e7c49d 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtils.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/CryptoUtils.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers; import org.bouncycastle.asn1.x509.Extension; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/IdentityDocumentService.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/IdentityDocumentService.java index 7878400964a..8a9137a491d 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/IdentityDocumentService.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/IdentityDocumentService.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.yahoo.vespa.defaults.Defaults; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceIdentity.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/InstanceIdentity.java index 20bbb2aa67e..d6e986959cb 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceIdentity.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/InstanceIdentity.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRefreshInformation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/InstanceRefreshInformation.java index dd893cb3143..d0c22d1d0d2 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRefreshInformation.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/InstanceRefreshInformation.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRegisterInformation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/InstanceRegisterInformation.java index e2355cb7a2d..dd9f164fef1 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRegisterInformation.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/InstanceRegisterInformation.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/SignedIdentityDocument.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/SignedIdentityDocument.java index 5d5b5430859..7bbd49c953f 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/SignedIdentityDocument.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/SignedIdentityDocument.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/package-info.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/package-info.java index 2d7cbbb6315..1b4842327dd 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/package-info.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/identityprovider/package-info.java @@ -3,6 +3,6 @@ * @author mortent */ @ExportPackage -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzIdentityProviderImplTest.java index 1ee23334a16..3a506a39c43 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/identityprovider/AthenzIdentityProviderImplTest.java @@ -1,14 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; -import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.RunnableWithTag; -import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.Scheduler; +import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; +import com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.RunnableWithTag; +import com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.Scheduler; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import org.junit.Test; +import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -23,15 +25,15 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.INITIAL_BACKOFF_DELAY; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.INITIAL_WAIT_NTOKEN; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.MAX_REGISTER_BACKOFF_DELAY; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.METRICS_UPDATER_TAG; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.REDUCED_UPDATE_PERIOD; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.REGISTER_INSTANCE_TAG; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.TIMEOUT_INITIAL_WAIT_TAG; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.UPDATE_CREDENTIALS_TAG; -import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.UPDATE_PERIOD; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.INITIAL_BACKOFF_DELAY; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.INITIAL_WAIT_NTOKEN; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.MAX_REGISTER_BACKOFF_DELAY; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.METRICS_UPDATER_TAG; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.REDUCED_UPDATE_PERIOD; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.REGISTER_INSTANCE_TAG; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.TIMEOUT_INITIAL_WAIT_TAG; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.UPDATE_CREDENTIALS_TAG; +import static com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl.UPDATE_PERIOD; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; @@ -46,77 +48,33 @@ public class AthenzIdentityProviderImplTest { private static final Metric DUMMY_METRIC = new Metric() { @Override - public void set(String s, Number number, Context context) {} + public void set(String s, Number number, Context context) { + } + @Override - public void add(String s, Number number, Context context) {} + public void add(String s, Number number, Context context) { + } + @Override - public Context createContext(Map<String, ?> stringMap) { return null; } + public Context createContext(Map<String, ?> stringMap) { + return null; + } }; private static final IdentityConfig IDENTITY_CONFIG = new IdentityConfig(new IdentityConfig.Builder() .service("tenantService").domain("tenantDomain").loadBalancerAddress("cfg")); - private final Set<String> IGNORED_TASKS = Stream.of(UPDATE_CREDENTIALS_TAG, METRICS_UPDATER_TAG) - .collect(Collectors.toSet()); - - @Test - public void athenz_credentials_are_retrieved_after_component_contruction_completed() { - IdentityDocumentService identityDocumentService = mock(IdentityDocumentService.class); - AthenzService athenzService = mock(AthenzService.class); - ManualClock clock = new ManualClock(Instant.EPOCH); - MockScheduler scheduler = new MockScheduler(clock); - - when(identityDocumentService.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); - when(athenzService.sendInstanceRegisterRequest(any(), any())).thenReturn( - new InstanceIdentity(null, "TOKEN")); - AthenzCredentialsService credentialService = - new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, clock); - - AthenzIdentityProvider identityProvider = - new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock); - - List<MockScheduler.CompletedTask> expectedTasks = - Arrays.asList( - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, Duration.ZERO), - new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN)); - // Don't run update credential tasks, otherwise infinite loop - List<MockScheduler.CompletedTask> completedTasks = - scheduler.runAllTasks(task -> !IGNORED_TASKS.contains(task.tag())); - assertEquals(expectedTasks, completedTasks); - assertEquals("TOKEN", identityProvider.getNToken()); - } - - @Test - public void register_instance_uses_exponential_backoff() { + @Test (expected = AthenzIdentityProviderException.class) + public void component_creation_fails_when_credentials_not_found() { AthenzCredentialsService credentialService = mock(AthenzCredentialsService.class); when(credentialService.registerInstance()) - .thenThrow(new RuntimeException("#1")) - .thenThrow(new RuntimeException("#2")) - .thenThrow(new RuntimeException("#3")) - .thenThrow(new RuntimeException("#4")) - .thenThrow(new RuntimeException("#5")) - .thenReturn(new AthenzCredentials("TOKEN", null, null, null, Instant.now())); + .thenThrow(new RuntimeException("athenz unavailable")); ManualClock clock = new ManualClock(Instant.EPOCH); MockScheduler scheduler = new MockScheduler(clock); AthenzIdentityProvider identityProvider = new AthenzIdentityProviderImpl(IDENTITY_CONFIG, DUMMY_METRIC, credentialService, scheduler, clock); - - List<MockScheduler.CompletedTask> expectedTasks = - Arrays.asList( - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, Duration.ZERO), - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY), - new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN), - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY.multipliedBy(2)), - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY.multipliedBy(4)), - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY.multipliedBy(8)), - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, MAX_REGISTER_BACKOFF_DELAY)); - // Don't run update credential tasks, otherwise infinite loop - List<MockScheduler.CompletedTask> completedTasks = - scheduler.runAllTasks(task -> !IGNORED_TASKS.contains(task.tag())); - assertEquals(expectedTasks, completedTasks); - assertEquals("TOKEN", identityProvider.getNToken()); } @Test @@ -125,6 +83,7 @@ public class AthenzIdentityProviderImplTest { AthenzService athenzService = mock(AthenzService.class); ManualClock clock = new ManualClock(Instant.EPOCH); MockScheduler scheduler = new MockScheduler(clock); + X509Certificate x509Certificate = mock(X509Certificate.class); when(identityDocumentService.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); when(athenzService.sendInstanceRegisterRequest(any(), any())).thenReturn( @@ -143,8 +102,6 @@ public class AthenzIdentityProviderImplTest { List<MockScheduler.CompletedTask> expectedTasks = Arrays.asList( - new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, Duration.ZERO), - new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN), new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD), new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD), new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, REDUCED_UPDATE_PERIOD), @@ -153,9 +110,8 @@ public class AthenzIdentityProviderImplTest { AtomicInteger counter = new AtomicInteger(0); List<MockScheduler.CompletedTask> completedTasks = scheduler.runAllTasks(task -> !task.tag().equals(METRICS_UPDATER_TAG) && - counter.getAndIncrement() < expectedTasks.size()); + counter.getAndIncrement() < expectedTasks.size()); assertEquals(expectedTasks, completedTasks); - assertEquals("TOKEN", identityProvider.getNToken()); } private static String getIdentityDocument() { @@ -232,7 +188,7 @@ public class AthenzIdentityProviderImplTest { if (o == null || getClass() != o.getClass()) return false; CompletedTask that = (CompletedTask) o; return Objects.equals(tag, that.tag) && - Objects.equals(delay, that.delay); + Objects.equals(delay, that.delay); } @Override @@ -243,9 +199,9 @@ public class AthenzIdentityProviderImplTest { @Override public String toString() { return "CompletedTask{" + - "tag='" + tag + '\'' + - ", delay=" + delay + - '}'; + "tag='" + tag + '\'' + + ", delay=" + delay + + '}'; } } } diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtilsTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/identityprovider/CryptoUtilsTest.java index dc9690355e8..0412b9071dd 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtilsTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/identityprovider/CryptoUtilsTest.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.athenz.impl; +package com.yahoo.vespa.hosted.athenz.identityprovider; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.junit.Test; diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index bc127da7a0b..13e49767964 100644 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -25,6 +25,6 @@ cmake3 \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ - -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ + -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm3.9/lib" \ ${EXTRA_CMAKE_ARGS} \ "${SOURCE_DIR}" diff --git a/build_settings.cmake b/build_settings.cmake index 2cccea9b64f..7616ff63ad9 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -81,6 +81,11 @@ else() set (VESPA_BOOST_LIB_SUFFIX "-mt-d") endif() +if(VESPA_USER) +else() + set(VESPA_USER "vespa") +endif() + if(EXTRA_INCLUDE_DIRECTORY) include_directories(SYSTEM ${EXTRA_INCLUDE_DIRECTORY}) endif() diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java index a0e93c2a9ad..ea638010ab7 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java @@ -29,31 +29,25 @@ public class ClusterStateView { private static Logger log = Logger.getLogger(ClusterStateView.class.getName()); private final ClusterState clusterState; private final ClusterStatsAggregator statsAggregator; - private final MetricUpdater metricUpdater; - /** - * @param metricUpdater may be null, in which case no stats will be reported. - */ - public static ClusterStateView create(String serializedClusterState, MetricUpdater metricUpdater) - throws ParseException { + public static ClusterStateView create(String serializedClusterState) throws ParseException { ClusterState clusterState = new ClusterState(serializedClusterState); - return new ClusterStateView(clusterState, createNewAggregator(clusterState, metricUpdater), metricUpdater); + return new ClusterStateView(clusterState, createNewAggregator(clusterState)); } - public static ClusterStateView create(final ClusterState clusterState, final MetricUpdater metricUpdater) { - return new ClusterStateView(clusterState, createNewAggregator(clusterState, metricUpdater), metricUpdater); + public static ClusterStateView create(final ClusterState clusterState) { + return new ClusterStateView(clusterState, createNewAggregator(clusterState)); } - private static ClusterStatsAggregator createNewAggregator(ClusterState clusterState, MetricUpdater metricUpdater) { + private static ClusterStatsAggregator createNewAggregator(ClusterState clusterState) { Set<Integer> upDistributors = getIndicesOfUpNodes(clusterState, NodeType.DISTRIBUTOR); Set<Integer> upStorageNodes = getIndicesOfUpNodes(clusterState, NodeType.STORAGE); - return new ClusterStatsAggregator(upDistributors, upStorageNodes, metricUpdater); + return new ClusterStatsAggregator(upDistributors, upStorageNodes); } - ClusterStateView(ClusterState clusterState, ClusterStatsAggregator statsAggregator, MetricUpdater metricUpdater) { + ClusterStateView(ClusterState clusterState, ClusterStatsAggregator statsAggregator) { this.clusterState = clusterState; this.statsAggregator = statsAggregator; - this.metricUpdater = metricUpdater; } /** @@ -85,8 +79,7 @@ public class ClusterStateView { ClusterState clonedClusterState = clusterState.clone(); return new ClusterStateView( clonedClusterState, - createNewAggregator(clonedClusterState, metricUpdater), - metricUpdater); + createNewAggregator(clonedClusterState)); } public ClusterState getClusterState() { return clusterState; } @@ -115,8 +108,8 @@ public class ClusterStateView { return; } - statsAggregator.updateForDistributor( - hostnames, node.getNodeIndex(), StorageNodeStatsBridge.generate(hostInfo.getDistributor())); + statsAggregator.updateForDistributor(node.getNodeIndex(), + StorageNodeStatsBridge.generate(hostInfo.getDistributor())); } public String toString() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java index 849a2aa23c2..3f7cd129fc1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java @@ -4,16 +4,13 @@ package com.yahoo.vespa.clustercontroller.core; import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.logging.Logger; - -import com.yahoo.log.LogLevel; /** - * A class that stores stats about outstanding merge operations for - * the current cluster state version, and exports metrics about these. + * Class that stores content cluster stats (with bucket space stats per node) for + * the current cluster state version. * - * Each distributor reports outstanding merge operations for the different - * storage nodes. These reports arrive with getnodestate RPC calls, + * Each distributor reports bucket space stats for the different content nodes. + * These reports arrive with getnodestate RPC calls, * and eventually ends up as calls to updateForDistributor(). * No assumptions are made on the sequence of getnodestate calls. * For instance, it's perfectly fine for the calls to arrive in the @@ -25,97 +22,55 @@ import com.yahoo.log.LogLevel; * distributor 2 * ... etc * - * Whereas the metrics we want, is how many merge operations are outstanding - * for a given storage nodes. So we need to keep track of the latest info - * from each distributor. - * * @author hakonhall */ public class ClusterStatsAggregator { - private static Logger log = Logger.getLogger(ClusterStatsAggregator.class.getName()); - private final Set<Integer> distributors; - private final MetricUpdater updater; - // Maps the distributor node index to a map of storage node index to the - // storage node's merge stats. - private final Map<Integer, StorageMergeStats> distributorToStats = new HashMap<>(); + // Maps the distributor node index to a map of content node index to the + // content node's stats. + private final Map<Integer, ContentClusterStats> distributorToStats = new HashMap<>(); - // This is only needed as an optimization. should just be the sum of distributorToStats' StorageMergeStats. - // Maps the storage node index to the aggregate merge stats for that storage node. + // This is only needed as an optimization. Is just the sum of distributorToStats' ContentClusterStats. + // Maps the content node index to the content node stats for that node. // This MUST be kept up-to-date with distributorToStats; - private final StorageMergeStats aggregatedStats; + private final ContentClusterStats aggregatedStats; - private int hostToStatsMapHashCode = 0; - - ClusterStatsAggregator(Set<Integer> distributors, Set<Integer> storageNodes, MetricUpdater updater) { + ClusterStatsAggregator(Set<Integer> distributors, Set<Integer> storageNodes) { this.distributors = distributors; - aggregatedStats = new StorageMergeStats(storageNodes); - this.updater = updater; + aggregatedStats = new ContentClusterStats(storageNodes); + } + + ContentClusterStats getAggregatedStats() { + return aggregatedStats; } /** * Update the aggregator with the newest available stats from a distributor. - * Will update metrics if necessary. */ - void updateForDistributor(Map<Integer, String> hostnames, int distributorIndex, StorageMergeStats storageStats) { + void updateForDistributor(int distributorIndex, ContentClusterStats clusterStats) { if (!distributors.contains(distributorIndex)) { return; } - - addStatsFromDistributor(distributorIndex, storageStats); - - if (distributorToStats.size() < distributors.size()) { - // Not all distributors have reported their merge stats through getnodestate yet. - return; - } - - Map<String, NodeMergeStats> hostToStatsMap = getHostToStatsMap(hostnames); - if (hostToStatsMap == null) { - return; - } - - if (hostToStatsMapHashCode == 0 || hostToStatsMapHashCode != hostToStatsMap.hashCode()) { - updater.updateMergeOpMetrics(hostToStatsMap); - hostToStatsMapHashCode = hostToStatsMap.hashCode(); - } - } - - private Map<String, NodeMergeStats> getHostToStatsMap(Map<Integer, String> hostnames) { - Map<String, NodeMergeStats> hostToStatsMap = new HashMap<>(aggregatedStats.size()); - for (NodeMergeStats nodeStats : aggregatedStats) { - // The hosts names are kept up-to-date from Slobrok, and MAY therefore be arbitrarily - // different from the node set used by aggregatedStats (and typically tied to a cluster state). - // If so, we will not pretend the returned map is complete, and will return null. - String host = hostnames.get(nodeStats.getNodeIndex()); - if (host == null) { - log.log(LogLevel.DEBUG, "Failed to find the host name of storage node " + nodeStats.getNodeIndex() + - ". Skipping the report from " + ClusterStatsAggregator.class.getName()); - return null; - } - - hostToStatsMap.put(host, nodeStats); - } - - return hostToStatsMap; + addStatsFromDistributor(distributorIndex, clusterStats); } - private void addStatsFromDistributor(int distributorIndex, StorageMergeStats storageStatsFromDistributor) { - StorageMergeStats previousStorageStats = distributorToStats.put(distributorIndex, storageStatsFromDistributor); + private void addStatsFromDistributor(int distributorIndex, ContentClusterStats clusterStats) { + ContentClusterStats prevClusterStats = distributorToStats.put(distributorIndex, clusterStats); - for (NodeMergeStats storageNode : aggregatedStats) { - Integer storageNodeIndex = storageNode.getNodeIndex(); + for (ContentNodeStats contentNode : aggregatedStats) { + Integer nodeIndex = contentNode.getNodeIndex(); - NodeMergeStats statsToAdd = storageStatsFromDistributor.getStorageNode(storageNodeIndex); + ContentNodeStats statsToAdd = clusterStats.getContentNode(nodeIndex); if (statsToAdd != null) { - storageNode.add(statsToAdd); + contentNode.add(statsToAdd); } - if (previousStorageStats != null) { - NodeMergeStats statsToSubtract = storageStatsFromDistributor.getStorageNode(storageNodeIndex); + if (prevClusterStats != null) { + ContentNodeStats statsToSubtract = prevClusterStats.getContentNode(nodeIndex); if (statsToSubtract != null) { - storageNode.subtract(statsToSubtract); + contentNode.subtract(statsToSubtract); } } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java new file mode 100644 index 00000000000..2698b079073 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentClusterStats.java @@ -0,0 +1,58 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import java.util.*; + +/** + * Class for storing pending content node stats for all content nodes in the cluster. + * + * @author hakonhall + */ +public class ContentClusterStats implements Iterable<ContentNodeStats> { + + // Maps a content node index to the content node's stats. + private final Map<Integer, ContentNodeStats> mapToNodeStats; + + public ContentClusterStats(Set<Integer> storageNodes) { + mapToNodeStats = new HashMap<>(storageNodes.size()); + for (Integer index : storageNodes) { + mapToNodeStats.put(index, new ContentNodeStats(index)); + } + } + + public ContentClusterStats(Map<Integer, ContentNodeStats> mapToNodeStats) { + this.mapToNodeStats = mapToNodeStats; + } + + @Override + public Iterator<ContentNodeStats> iterator() { + return mapToNodeStats.values().iterator(); + } + + ContentNodeStats getContentNode(Integer index) { + return mapToNodeStats.get(index); + } + + int size() { + return mapToNodeStats.size(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContentClusterStats that = (ContentClusterStats) o; + return Objects.equals(mapToNodeStats, that.mapToNodeStats); + } + + @Override + public int hashCode() { + return Objects.hash(mapToNodeStats); + } + + @Override + public String toString() { + return String.format("{mapToNodeStats=[%s]}", + Arrays.toString(mapToNodeStats.entrySet().toArray())); + } +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java new file mode 100644 index 00000000000..cefb3c3c31f --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStats.java @@ -0,0 +1,139 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNode; + +import java.util.*; + +/** + * @author hakonhall + */ +public class ContentNodeStats { + + private int nodeIndex; + private Map<String, BucketSpaceStats> bucketSpaces = new HashMap<>(); + + public static class BucketSpaceStats { + private long bucketsTotal; + private long bucketsPending; + + private BucketSpaceStats() { + this.bucketsTotal = 0; + this.bucketsPending = 0; + } + + private BucketSpaceStats(long bucketsTotal, long bucketsPending) { + this.bucketsTotal = bucketsTotal; + this.bucketsPending = bucketsPending; + } + + public static BucketSpaceStats empty() { + return new BucketSpaceStats(); + } + + public static BucketSpaceStats of(long bucketsTotal, long bucketsPending) { + return new BucketSpaceStats(bucketsTotal, bucketsPending); + } + + public long getBucketsTotal() { + return bucketsTotal; + } + + public long getBucketsPending() { + return bucketsPending; + } + + public void merge(BucketSpaceStats rhs, int factor) { + this.bucketsTotal += (factor * rhs.bucketsTotal); + this.bucketsPending += (factor * rhs.bucketsPending); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BucketSpaceStats that = (BucketSpaceStats) o; + return bucketsTotal == that.bucketsTotal && + bucketsPending == that.bucketsPending; + } + + @Override + public int hashCode() { + return Objects.hash(bucketsTotal, bucketsPending); + } + + @Override + public String toString() { + return "{bucketsTotal=" + bucketsTotal + ", bucketsPending=" + bucketsPending + "}"; + } + } + + public ContentNodeStats(StorageNode storageNode) { + this.nodeIndex = storageNode.getIndex(); + for (StorageNode.BucketSpaceStats stats : storageNode.getBucketSpacesStats()) { + if (stats.valid()) { + this.bucketSpaces.put(stats.getName(), + BucketSpaceStats.of(stats.getBucketStats().getTotal(), + stats.getBucketStats().getPending())); + } else { + this.bucketSpaces.put(stats.getName(), BucketSpaceStats.empty()); + } + } + } + + public ContentNodeStats(int index) { + this(index, new HashMap<>()); + } + + public ContentNodeStats(int index, Map<String, BucketSpaceStats> bucketSpaces) { + this.nodeIndex = index; + this.bucketSpaces = bucketSpaces; + } + + public int getNodeIndex() { return nodeIndex; } + + public void add(ContentNodeStats stats) { + merge(stats, 1); + } + + public void subtract(ContentNodeStats stats) { + merge(stats, -1); + } + + private void merge(ContentNodeStats stats, int factor) { + for (Map.Entry<String, BucketSpaceStats> entry : stats.bucketSpaces.entrySet()) { + BucketSpaceStats statsToUpdate = bucketSpaces.get(entry.getKey()); + if (statsToUpdate == null && factor == 1) { + statsToUpdate = new BucketSpaceStats(); + bucketSpaces.put(entry.getKey(), statsToUpdate); + } + if (statsToUpdate != null) { + statsToUpdate.merge(entry.getValue(), factor); + } + } + } + + public Map<String, BucketSpaceStats> getBucketSpaces() { + return bucketSpaces; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContentNodeStats that = (ContentNodeStats) o; + return nodeIndex == that.nodeIndex && + Objects.equals(bucketSpaces, that.bucketSpaces); + } + + @Override + public int hashCode() { + return Objects.hash(nodeIndex, bucketSpaces); + } + + @Override + public String toString() { + return String.format("{index=%d, bucketSpaces=[%s]}", + nodeIndex, Arrays.toString(bucketSpaces.entrySet().toArray())); + } +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index a5419c64818..ba56d75ab4c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -122,7 +122,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd this.stateGatherer = nodeStateGatherer; this.stateChangeHandler = stateChangeHandler; this.systemStateBroadcaster = systemStateBroadcaster; - this.stateVersionTracker = new StateVersionTracker(metricUpdater); + this.stateVersionTracker = new StateVersionTracker(); this.metricUpdater = metricUpdater; this.statusPageServer = statusPage; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index b3b10e1d0d8..199e9a3169b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -85,7 +85,4 @@ public class MetricUpdater { metricReporter.add("node-event", 1); } - public void updateMergeOpMetrics(Map<String, NodeMergeStats> storageNodeStats) { - // TODO(hakonhall): Remove this method once we figure out how to propagate metrics to state HTTP API. - } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java deleted file mode 100644 index 97112e01aed..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.clustercontroller.core; - -import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNode; - -/** - * @author hakonhall - */ -public class NodeMergeStats { - - /** - * Constructor that sets values to zero if not present. - */ - public NodeMergeStats(StorageNode storageNodePojo) { - this.nodeIndex = storageNodePojo.getIndex(); - - StorageNode.OutstandingMergeOps mergeOps = storageNodePojo.getOutstandingMergeOpsOrNull(); - if (mergeOps == null) { - mergeOps = new StorageNode.OutstandingMergeOps(); - } - syncing = createAmount(mergeOps.getSyncingOrNull()); - copyingIn = createAmount(mergeOps.getCopyingInOrNull()); - movingOut = createAmount(mergeOps.getMovingOutOrNull()); - copyingOut = createAmount(mergeOps.getCopyingOutOrNull()); - } - - private static Amount createAmount(StorageNode.Buckets bucketOrNull) { - if (bucketOrNull == null) { - return new Amount(); - } - return new Amount(bucketOrNull.getBuckets()); - } - - static public class Amount { - private long buckets; - - Amount() { this(0); } - Amount(long buckets) { this.buckets = buckets; } - - public void set(Amount other) { - buckets = other.buckets; - } - - public long getBuckets() { - return buckets; - } - - /** - * Logically, add (factor * amount) to this object. - */ - void scaledAdd(int factor, Amount amount) { - buckets += factor * amount.buckets; - } - - public boolean equals(Object other) { - if (!(other instanceof Amount)) { - return false; - } - Amount otherAmount = (Amount) other; - return buckets == otherAmount.buckets; - } - - public int hashCode() { - return (int)buckets; - } - - public String toString() { - return String.format("{buckets = %d}", buckets); - } - } - - private final Amount syncing; - private final Amount copyingIn; - private final Amount movingOut; - private final Amount copyingOut; - private int nodeIndex; - - /** - * An instance with all 0 amounts. - */ - public NodeMergeStats(int index) { - this(index, new Amount(), new Amount(), new Amount(), new Amount()); - } - - NodeMergeStats(int index, Amount syncing, Amount copyingIn, Amount movingOut, Amount copyingOut) { - this.nodeIndex = index; - this.syncing = syncing; - this.copyingIn = copyingIn; - this.movingOut = movingOut; - this.copyingOut = copyingOut; - } - - public void set(NodeMergeStats stats) { - nodeIndex = stats.nodeIndex; - syncing.set(stats.syncing); - copyingIn.set(stats.copyingIn); - movingOut.set(stats.movingOut); - copyingOut.set(stats.copyingOut); - } - - int getNodeIndex() { return nodeIndex; } - public Amount getSyncing() { return syncing; } - public Amount getCopyingIn() { return copyingIn; } - public Amount getMovingOut() { return movingOut; } - public Amount getCopyingOut() { return copyingOut; } - - void add(NodeMergeStats stats) { - scaledAdd(1, stats); - } - - void subtract(NodeMergeStats stats) { - scaledAdd(-1, stats); - } - - /** - * Logically, adds (factor * stats) to this object. factor of 1 is normal add, -1 is subtraction. - */ - private void scaledAdd(int factor, NodeMergeStats stats) { - syncing.scaledAdd(factor, stats.syncing); - copyingIn.scaledAdd(factor, stats.copyingIn); - movingOut.scaledAdd(factor, stats.movingOut); - copyingOut.scaledAdd(factor, stats.copyingOut); - } - - @Override - public int hashCode() { - return (int) (syncing.buckets + - copyingIn.buckets * 31 + - movingOut.buckets * 17 + - copyingOut.buckets * 7); - } - - @Override - public boolean equals(Object other) { - if (!(other instanceof NodeMergeStats)) { - return false; - } - - NodeMergeStats otherStats = (NodeMergeStats) other; - return nodeIndex == otherStats.nodeIndex && - syncing.equals(otherStats.syncing) && - copyingIn.equals(otherStats.copyingIn) && - movingOut.equals(otherStats.movingOut) && - copyingOut.equals(otherStats.copyingOut); - } - - public String toString() { - return String.format("{index = %d, syncing = %s, copyingIn = %s, movingOut = %s, copyingOut = %s}", - nodeIndex, syncing, copyingIn, movingOut, copyingOut); - } -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java index 518361f18fc..902189fca8b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateVersionTracker.java @@ -32,15 +32,13 @@ public class StateVersionTracker { private AnnotatedClusterState latestCandidateState = AnnotatedClusterState.emptyState(); private AnnotatedClusterState currentClusterState = latestCandidateState; - private final MetricUpdater metricUpdater; private ClusterStateView clusterStateView; private final LinkedList<ClusterStateHistoryEntry> clusterStateHistory = new LinkedList<>(); private int maxHistoryEntryCount = 50; - StateVersionTracker(final MetricUpdater metricUpdater) { - this.metricUpdater = metricUpdater; - clusterStateView = ClusterStateView.create(currentUnversionedState, metricUpdater); + StateVersionTracker() { + clusterStateView = ClusterStateView.create(currentUnversionedState); } void setVersionRetrievedFromZooKeeper(final int version) { @@ -121,7 +119,7 @@ public class StateVersionTracker { lowestObservedDistributionBits, newState.getClusterState().getDistributionBitCount()); // TODO should this take place in updateLatestCandidateState instead? I.e. does it require a consolidated state? - clusterStateView = ClusterStateView.create(currentClusterState.getClusterState(), metricUpdater); + clusterStateView = ClusterStateView.create(currentClusterState.getClusterState()); } private void recordCurrentStateInHistoryAtTime(final long currentTimeMs) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java deleted file mode 100644 index 6a44bee8cce..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.clustercontroller.core; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -/** - * Class for storing the pending merge operation stats for all the storage nodes. - * - * @author hakonhall - */ -public class StorageMergeStats implements Iterable<NodeMergeStats> { - - // Maps a storage node index to the storage node's pending merges stats. - private final Map<Integer, NodeMergeStats> mapToNodeStats; - - public StorageMergeStats(Set<Integer> storageNodes) { - mapToNodeStats = new HashMap<>(storageNodes.size()); - for (Integer index : storageNodes) { - mapToNodeStats.put(index, new NodeMergeStats(index)); - } - } - - public StorageMergeStats(Map<Integer, NodeMergeStats> mapToNodeStats) { - this.mapToNodeStats = mapToNodeStats; - } - - @Override - public Iterator<NodeMergeStats> iterator() { - return mapToNodeStats.values().iterator(); - } - - NodeMergeStats getStorageNode(Integer index) { - return mapToNodeStats.get(index); - } - - int size() { - return mapToNodeStats.size(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof StorageMergeStats)) { - return false; - } - - StorageMergeStats that = (StorageMergeStats) o; - - if (mapToNodeStats != null ? !mapToNodeStats.equals(that.mapToNodeStats) : that.mapToNodeStats != null) { - return false; - } - return true; - } - - @Override - public int hashCode() { - return mapToNodeStats != null ? mapToNodeStats.hashCode() : 0; - } - -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java index 9e951236c45..bf7c1fad6ca 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.clustercontroller.core.hostinfo; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.List; + /** * Keeping information about a storage node seen from the distributor. * @@ -36,31 +39,43 @@ public class StorageNode { public Put getPut() { return put; } } - static public class Buckets { - private final long buckets; + static public class BucketStats { + private final long total; + private final long pending; @JsonCreator - public Buckets(@JsonProperty("buckets") Long buckets) { - this.buckets = buckets; + public BucketStats(@JsonProperty("total") Long total, @JsonProperty("pending") Long pending) { + this.total = total; + this.pending = pending; } - public long getBuckets() { return buckets; } + public long getTotal() { + return total; + } + public long getPending() { + return pending; + } } - static public class OutstandingMergeOps { - @JsonProperty("syncing") - private Buckets syncing; - @JsonProperty("copying-in") - private Buckets copyingIn; - @JsonProperty("moving-out") - private Buckets movingOut; - @JsonProperty("copying-out") - private Buckets copyingOut; - - public Buckets getSyncingOrNull() { return syncing; } - public Buckets getCopyingInOrNull() { return copyingIn; } - public Buckets getMovingOutOrNull() { return movingOut; } - public Buckets getCopyingOutOrNull() { return copyingOut; } + static public class BucketSpaceStats { + private final String name; + @JsonProperty("buckets") + private BucketStats bucketStats = null; + + @JsonCreator + public BucketSpaceStats(@JsonProperty("name") String name) { + this.name = name; + } + + public String getName() { + return name; + } + public boolean valid() { + return bucketStats != null; + } + public BucketStats getBucketStats() { + return bucketStats; + } } private final Integer index; @@ -74,8 +89,8 @@ public class StorageNode { @JsonProperty("min-current-replication-factor") private Integer minCurrentReplicationFactor; - @JsonProperty("outstanding-merge-ops") - private OutstandingMergeOps outstandingMergeOps; + @JsonProperty("bucket-spaces") + private List<BucketSpaceStats> bucketSpacesStats = new ArrayList<>(); @JsonCreator public StorageNode(@JsonProperty("node-index") Integer index) { @@ -95,8 +110,7 @@ public class StorageNode { return minCurrentReplicationFactor; } - public OutstandingMergeOps getOutstandingMergeOpsOrNull() { - return outstandingMergeOps; + public List<BucketSpaceStats> getBucketSpacesStats() { + return bucketSpacesStats; } - } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java index 6d0da8a9fba..55b7e4bb8c1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java @@ -1,11 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.hostinfo; -import com.yahoo.vespa.clustercontroller.core.LatencyStats; -import com.yahoo.vespa.clustercontroller.core.NodeMergeStats; -import com.yahoo.vespa.clustercontroller.core.StorageMergeStats; -import com.yahoo.vespa.clustercontroller.core.StorageNodeStats; -import com.yahoo.vespa.clustercontroller.core.StorageNodeStatsContainer; +import com.yahoo.vespa.clustercontroller.core.*; import java.util.HashMap; import java.util.List; @@ -45,12 +41,12 @@ public class StorageNodeStatsBridge { return container; } - public static StorageMergeStats generate(Distributor distributor) { - Map<Integer, NodeMergeStats> mapToNodeStats = new HashMap<>(); + public static ContentClusterStats generate(Distributor distributor) { + Map<Integer, ContentNodeStats> mapToNodeStats = new HashMap<>(); for (StorageNode storageNode : distributor.getStorageNodes()) { - mapToNodeStats.put(storageNode.getIndex(), new NodeMergeStats(storageNode)); + mapToNodeStats.put(storageNode.getIndex(), new ContentNodeStats(storageNode)); } - return new StorageMergeStats(mapToNodeStats); + return new ContentClusterStats(mapToNodeStats); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java index 3b04fec3e79..dc8a4a0d441 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java @@ -22,10 +22,8 @@ public class ClusterStateViewTest { final NodeInfo nodeInfo = mock(NodeInfo.class); final Node node = mock(Node.class); final ClusterStatsAggregator statsAggregator = mock(ClusterStatsAggregator.class); - final StorageMergeStats storageStats = mock(StorageMergeStats.class); final ClusterState clusterState = mock(ClusterState.class); - final MetricUpdater metricUpdater = mock(MetricUpdater.class); - final ClusterStateView clusterStateView = new ClusterStateView(clusterState, statsAggregator, metricUpdater); + final ClusterStateView clusterStateView = new ClusterStateView(clusterState, statsAggregator); HostInfo createHostInfo(String version) { return HostInfo.createHostInfo("{ \"cluster-state-version\": " + version + " }"); @@ -37,7 +35,7 @@ public class ClusterStateViewTest { clusterStateView.handleUpdatedHostInfo(hostnames, nodeInfo, createHostInfo("101")); - verify(statsAggregator, never()).updateForDistributor(any(), anyInt(), any()); + verify(statsAggregator, never()).updateForDistributor(anyInt(), any()); } @@ -49,7 +47,7 @@ public class ClusterStateViewTest { clusterStateView.handleUpdatedHostInfo(hostnames, nodeInfo, createHostInfo("22")); - verify(statsAggregator, never()).updateForDistributor(any(), anyInt(), any()); + verify(statsAggregator, never()).updateForDistributor(anyInt(), any()); } @Test @@ -59,7 +57,7 @@ public class ClusterStateViewTest { clusterStateView.handleUpdatedHostInfo(hostnames, nodeInfo, createHostInfo("22")); - verify(statsAggregator, never()).updateForDistributor(any(), anyInt(), any()); + verify(statsAggregator, never()).updateForDistributor(anyInt(), any()); } @Test @@ -81,8 +79,7 @@ public class ClusterStateViewTest { clusterStateView.handleUpdatedHostInfo(hostnames, nodeInfo, hostInfo); - verify(statsAggregator).updateForDistributor( - hostnames, 3, StorageNodeStatsBridge.generate(hostInfo.getDistributor())); + verify(statsAggregator).updateForDistributor(3, StorageNodeStatsBridge.generate(hostInfo.getDistributor())); } @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java index eb4455b9492..c92d414aac8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java @@ -1,217 +1,142 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.google.common.collect.Sets; import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; import java.util.*; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; /** * @author hakonhall * @since 5.34 */ -@RunWith(MockitoJUnitRunner.class) public class ClusterStatsAggregatorTest { - final Set<Integer> distributors = new HashSet<>(); - final Set<Integer> storageNodes = new HashSet<>(); - final Map<Integer, String> hostnames = new HashMap<>(); - final MetricUpdater updater = mock(MetricUpdater.class); - StorageMergeStats storageStats; + private static class StatsBuilder { + private final Map<Integer, Map<String, ContentNodeStats.BucketSpaceStats> > stats = new HashMap<>(); - private void addDistributors(Integer... indices) { - for (Integer i : indices) { - distributors.add(i); + public StatsBuilder add(int nodeIndex, String bucketSpace, long bucketsTotal, long bucketsPending) { + return add(nodeIndex, bucketSpace, ContentNodeStats.BucketSpaceStats.of(bucketsTotal, bucketsPending)); } - } - - private static class StorageNodeSpec { - public StorageNodeSpec(Integer index, String hostname) { - this.index = index; - this.hostname = hostname; + public StatsBuilder add(int nodeIndex, String bucketSpace) { + return add(nodeIndex, bucketSpace, ContentNodeStats.BucketSpaceStats.empty()); + } + public StatsBuilder add(int nodeIndex, String bucketSpace, ContentNodeStats.BucketSpaceStats bucketSpaceStats) { + Map<String, ContentNodeStats.BucketSpaceStats> contentNodeStats = stats.get(nodeIndex); + if (contentNodeStats == null) { + contentNodeStats = new HashMap<>(); + stats.put(nodeIndex, contentNodeStats); + } + contentNodeStats.put(bucketSpace, bucketSpaceStats); + return this; + } + public StatsBuilder add(int nodeIndex) { + stats.put(nodeIndex, new HashMap<>()); + return this; + } + public ContentClusterStats build() { + Map<Integer, ContentNodeStats> nodeToStatsMap = new HashMap<>(); + stats.forEach((nodeIndex, bucketSpaces) -> + nodeToStatsMap.put(nodeIndex, new ContentNodeStats(nodeIndex, bucketSpaces))); + return new ContentClusterStats(nodeToStatsMap); } - public Integer index; - public String hostname; } - private void addStorageNodes(StorageNodeSpec... specs) { - for (StorageNodeSpec spec : specs) { - storageNodes.add(spec.index); - hostnames.put(spec.index, spec.hostname); + private static class Fixture { + private ClusterStatsAggregator aggregator; + public Fixture(Set<Integer> distributorNodes, + Set<Integer> contentNodes) { + aggregator = new ClusterStatsAggregator(distributorNodes, contentNodes); + } + public void update(int distributorIndex, StatsBuilder clusterStats) { + aggregator.updateForDistributor(distributorIndex, clusterStats.build()); + } + public void verify(StatsBuilder expectedStats) { + assertEquals(expectedStats.build(), aggregator.getAggregatedStats()); } - storageStats = new StorageMergeStats(storageNodes); } - private void putStorageStats(int index, int syncing, int copyingIn, int movingOut, int copyingOut) { - storageStats.getStorageNode(index).set(createStats(index, syncing, copyingIn, movingOut, copyingOut)); + private static Set<Integer> distributorNodes(Integer... indices) { + return Sets.newHashSet(indices); } - private static NodeMergeStats createStats(int index, int syncing, int copyingIn, int movingOut, int copyingOut) { - return new NodeMergeStats( - index, - new NodeMergeStats.Amount(syncing), - new NodeMergeStats.Amount(copyingIn), - new NodeMergeStats.Amount(movingOut), - new NodeMergeStats.Amount(copyingOut)); + private static Set<Integer> contentNodes(Integer... indices) { + return Sets.newHashSet(indices); } @Test - public void testSimple() { - final int distributorIndex = 1; - addDistributors(distributorIndex); - - final int storageNodeIndex = 11; - addStorageNodes(new StorageNodeSpec(storageNodeIndex, "storage-node")); - - putStorageStats(storageNodeIndex, 5, 6, 7, 8); - - ClusterStatsAggregator aggregator = new ClusterStatsAggregator(distributors, storageNodes, updater); - aggregator.updateForDistributor(hostnames, distributorIndex, storageStats); - - Map<String, NodeMergeStats> expectedStorageNodeStats = new HashMap<>(); - expectedStorageNodeStats.put("storage-node", createStats(storageNodeIndex, 5, 6, 7, 8)); - - verify(updater).updateMergeOpMetrics(expectedStorageNodeStats); + public void aggregator_handles_updates_to_single_distributor_and_content_node() { + Fixture f = new Fixture(distributorNodes(1), contentNodes(3)); + StatsBuilder stats = new StatsBuilder() + .add(3, "default", 10, 1) + .add(3, "global", 11, 2); + f.update(1, stats); + f.verify(stats); } @Test - public void testComplex() { - final int distributor1 = 1; - final int distributor2 = 2; - addDistributors(distributor1, distributor2); - - final int storageNode1 = 11; - final int storageNode2 = 12; - addStorageNodes( - new StorageNodeSpec(storageNode1, "storage-node-1"), - new StorageNodeSpec(storageNode2, "storage-node-2")); - - ClusterStatsAggregator aggregator = new ClusterStatsAggregator(distributors, storageNodes, updater); - - // Distributor 1. - putStorageStats(storageNode1, 0, 1, 2, 3); - putStorageStats(storageNode2, 20, 21, 22, 23); - aggregator.updateForDistributor(hostnames, distributor1, storageStats); - - // Distributor 2. - putStorageStats(storageNode1, 10, 11, 12, 13); - putStorageStats(storageNode2, 30, 31, 32, 33); - aggregator.updateForDistributor(hostnames, distributor2, storageStats); - - Map<String, NodeMergeStats> expectedStorageNodeStats = new HashMap<>(); - expectedStorageNodeStats.put("storage-node-1", createStats(storageNode1, 0 + 10, 1 + 11, 2 + 12, 3 + 13)); - expectedStorageNodeStats.put("storage-node-2", createStats(storageNode2, 20 + 30, 21 + 31, 22 + 32, 23 + 33)); - - verify(updater, times(1)).updateMergeOpMetrics(expectedStorageNodeStats); + public void aggregator_handles_updates_to_multiple_distributors_and_content_nodes() { + Fixture f = new Fixture(distributorNodes(1, 2), contentNodes(3, 4)); + + f.update(1, new StatsBuilder() + .add(3, "default", 10, 1) + .add(3, "global", 11, 2) + .add(4, "default", 12, 3) + .add(4, "global", 13, 4)); + f.update(2, new StatsBuilder() + .add(3, "default", 14, 5) + .add(3, "global", 15, 6) + .add(4, "default", 16, 7) + .add(4, "global", 17, 8)); + f.verify(new StatsBuilder() + .add(3, "default", 10 + 14, 1 + 5) + .add(3, "global", 11 + 15, 2 + 6) + .add(4, "default", 12 + 16, 3 + 7) + .add(4, "global", 13 + 17, 4 + 8)); } @Test - public void testHashCodeCache() { - final int distributor1 = 1; - final int distributor2 = 2; - addDistributors(distributor1, distributor2); + public void aggregator_handles_multiple_updates_from_same_distributor() { + Fixture f = new Fixture(distributorNodes(1, 2), contentNodes(3)); - final int storageNode1 = 11; - final int storageNode2 = 12; - addStorageNodes( - new StorageNodeSpec(storageNode1, "storage-node-1"), - new StorageNodeSpec(storageNode2, "storage-node-2")); + f.update(1, new StatsBuilder().add(3, "default")); + f.verify(new StatsBuilder().add(3, "default")); - ClusterStatsAggregator aggregator = new ClusterStatsAggregator(distributors, storageNodes, updater); + f.update(2, new StatsBuilder().add(3, "default", 10, 1)); + f.verify(new StatsBuilder().add(3, "default", 10, 1)); - // Distributor 1. - putStorageStats(storageNode1, 0, 1, 2, 3); - putStorageStats(storageNode2, 20, 21, 22, 23); - aggregator.updateForDistributor(hostnames, distributor1, storageStats); + f.update(1, new StatsBuilder().add(3, "default", 11, 2)); + f.verify(new StatsBuilder().add(3, "default", 10 + 11, 1 + 2)); - // Distributor 2. - putStorageStats(storageNode1, 10, 11, 12, 13); - putStorageStats(storageNode2, 30, 31, 32, 33); - aggregator.updateForDistributor(hostnames, distributor2, storageStats); + f.update(2, new StatsBuilder().add(3, "default", 15, 6)); + f.verify(new StatsBuilder().add(3, "default", 11 + 15, 2 + 6)); - // If we add call another updateForDistributor with the same arguments, updateMergeOpMetrics() should not be called. - // See times(1) below. - aggregator.updateForDistributor(hostnames, distributor2, storageStats); + f.update(1, new StatsBuilder().add(3, "default", 16, 7)); + f.verify(new StatsBuilder().add(3, "default", 15 + 16, 6 + 7)); - Map<String, NodeMergeStats> expectedStorageNodeStats = new HashMap<>(); - expectedStorageNodeStats.put("storage-node-1", createStats(storageNode1, 0 + 10, 1 + 11, 2 + 12, 3 + 13)); - expectedStorageNodeStats.put("storage-node-2", createStats(storageNode2, 20 + 30, 21 + 31, 22 + 32, 23 + 33)); - - - verify(updater, times(1)).updateMergeOpMetrics(expectedStorageNodeStats); + f.update(2, new StatsBuilder().add(3, "default", 12, 3)); + f.verify(new StatsBuilder().add(3, "default", 16 + 12, 7 + 3)); } @Test - public void testUnknownDistributor() { - final int upDistributor = 1; - final int DownDistributorIndex = 2; - addDistributors(upDistributor); - - final int storageNodeIndex = 11; - addStorageNodes(new StorageNodeSpec(storageNodeIndex, "storage-node")); - - putStorageStats(storageNodeIndex, 5, 6, 7, 8); - - ClusterStatsAggregator aggregator = new ClusterStatsAggregator(distributors, storageNodes, updater); - aggregator.updateForDistributor(hostnames, DownDistributorIndex, storageStats); - - verify(updater, never()).updateMergeOpMetrics(any()); + public void aggregator_handles_more_content_nodes_that_distributors() { + Fixture f = new Fixture(distributorNodes(1), contentNodes(3, 4)); + StatsBuilder stats = new StatsBuilder() + .add(3, "default", 10, 1) + .add(4, "default", 11, 2); + f.update(1, stats); + f.verify(stats); } @Test - public void testMoreStorageNodesThanDistributors() { - final int distributor1 = 1; - addDistributors(distributor1); - - final int storageNode1 = 11; - final int storageNode2 = 12; - addStorageNodes( - new StorageNodeSpec(storageNode1, "storage-node-1"), - new StorageNodeSpec(storageNode2, "storage-node-2")); - - ClusterStatsAggregator aggregator = new ClusterStatsAggregator(distributors, storageNodes, updater); - - // Distributor 1. - putStorageStats(storageNode1, 0, 1, 2, 3); - putStorageStats(storageNode2, 20, 21, 22, 23); - aggregator.updateForDistributor(hostnames, distributor1, storageStats); - - Map<String, NodeMergeStats> expectedStorageNodeStats = new HashMap<>(); - expectedStorageNodeStats.put("storage-node-1", createStats(storageNode1, 0, 1, 2, 3)); - expectedStorageNodeStats.put("storage-node-2", createStats(storageNode2, 20, 21, 22, 23)); - - verify(updater, times(1)).updateMergeOpMetrics(expectedStorageNodeStats); + public void aggregator_ignores_updates_to_unknown_distributor() { + Fixture f = new Fixture(distributorNodes(1), contentNodes(3)); + final int downDistributorIndex = 2; + f.update(downDistributorIndex, new StatsBuilder() + .add(3, "default", 7, 3)); + f.verify(new StatsBuilder().add(3)); } - @Test - public void testMoreDistributorsThanStorageNodes() { - final int distributor1 = 1; - final int distributor2 = 2; - addDistributors(distributor1, distributor2); - - final int storageNode1 = 11; - addStorageNodes(new StorageNodeSpec(storageNode1, "storage-node-1")); - - ClusterStatsAggregator aggregator = new ClusterStatsAggregator(distributors, storageNodes, updater); - - // Distributor 1. - putStorageStats(storageNode1, 0, 1, 2, 3); - aggregator.updateForDistributor(hostnames, distributor1, storageStats); - - // Distributor 2. - putStorageStats(storageNode1, 10, 11, 12, 13); - aggregator.updateForDistributor(hostnames, distributor2, storageStats); - - Map<String, NodeMergeStats> expectedStorageNodeStats = new HashMap<>(); - expectedStorageNodeStats.put("storage-node-1", createStats(storageNode1, 0 + 10, 1 + 11, 2 + 12, 3 + 13)); - - verify(updater, times(1)).updateMergeOpMetrics(expectedStorageNodeStats); - } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java index 0af7bcbfeaf..d27acf8bc17 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateVersionTrackerTest.java @@ -11,14 +11,11 @@ import org.junit.Test; import java.util.Arrays; import java.util.Optional; -import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; public class StateVersionTrackerTest { @@ -28,7 +25,7 @@ public class StateVersionTrackerTest { } private static StateVersionTracker createWithMockedMetrics() { - return new StateVersionTracker(mock(MetricUpdater.class)); + return new StateVersionTracker(); } private static void updateAndPromote(final StateVersionTracker versionTracker, diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java index 8a17fedee86..5319d741503 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.hostinfo; -import com.yahoo.vespa.clustercontroller.core.NodeMergeStats; -import com.yahoo.vespa.clustercontroller.core.StorageMergeStats; +import com.yahoo.vespa.clustercontroller.core.ContentNodeStats; +import com.yahoo.vespa.clustercontroller.core.ContentClusterStats; import com.yahoo.vespa.clustercontroller.core.StorageNodeStats; import com.yahoo.vespa.clustercontroller.core.StorageNodeStatsContainer; import org.junit.Test; @@ -12,11 +12,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Iterator; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; /** * @author hakonhall @@ -49,19 +49,30 @@ public class StorageNodeStatsBridgeTest { } @Test - public void testStorageMergeStats() throws IOException { + public void testContentNodeStats() throws IOException { String data = getJsonString(); HostInfo hostInfo = HostInfo.createHostInfo(data); - StorageMergeStats storageMergeStats = StorageNodeStatsBridge.generate(hostInfo.getDistributor()); - int size = 0; - for (NodeMergeStats mergeStats : storageMergeStats) { - assertThat(mergeStats.getCopyingIn().getBuckets(), is(2L)); - assertThat(mergeStats.getCopyingOut().getBuckets(), is(4L)); - assertThat(mergeStats.getSyncing().getBuckets(), is(1L)); - assertThat(mergeStats.getMovingOut().getBuckets(), is(3L)); - size++; + ContentClusterStats clusterStats = StorageNodeStatsBridge.generate(hostInfo.getDistributor()); + Iterator<ContentNodeStats> itr = clusterStats.iterator(); + { // content node 0 + ContentNodeStats stats = itr.next(); + assertThat(stats.getNodeIndex(), is(0)); + assertThat(stats.getBucketSpaces().size(), is(2)); + assertBucketSpaceStats(11, 3, stats.getBucketSpaces().get("default")); + assertBucketSpaceStats(13, 5, stats.getBucketSpaces().get("global")); } - assertThat(size, is(2)); + { // content node 1 + ContentNodeStats stats = itr.next(); + assertThat(stats.getNodeIndex(), is(1)); + assertThat(stats.getBucketSpaces().size(), is(1)); + assertBucketSpaceStats(0, 0, stats.getBucketSpaces().get("default")); + } + assertFalse(itr.hasNext()); + } + + private static void assertBucketSpaceStats(long expBucketsTotal, long expBucketsPending, ContentNodeStats.BucketSpaceStats stats) { + assertThat(stats.getBucketsTotal(), is(expBucketsTotal)); + assertThat(stats.getBucketsPending(), is(expBucketsPending)); } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 391d284a325..9bfcd4ecb6d 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -8,7 +8,6 @@ import com.yahoo.config.application.XmlPreProcessor; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.RuleConfigDeriver; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; import com.yahoo.config.application.api.ApplicationFile; @@ -669,11 +668,6 @@ public class FilesApplicationPackage implements ApplicationPackage { } @Override - public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { - return preprocess(zone, logger); - } - - @Override public ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { IOUtils.recursiveDeleteDir(preprocessedDir); IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(preprocessed) && diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index c2f6dbcbf4b..fe6f7da2092 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -271,23 +271,6 @@ public interface ApplicationPackage { * will not mutate the existing application package. * * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove - * @param ruleConfigDeriver ignored - * @param logger A {@link DeployLogger} to add output that will be returned to the user - * - * @return A new application package instance pointing to a new location - */ - // TODO: Remove when last version in use is 6.202 - default ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) - throws IOException, TransformerException, ParserConfigurationException, SAXException { - throw new UnsupportedOperationException("This application package does not support preprocessing"); - } - - /** - * Preprocess an application for a given zone and return a new application package pointing to the preprocessed - * application package. This is the entry point for the multi environment application package support. This method - * will not mutate the existing application package. - * - * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove * @param logger A {@link DeployLogger} to add output that will be returned to the user * * @return A new application package instance pointing to a new location diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/RuleConfigDeriver.java b/config-model-api/src/main/java/com/yahoo/config/application/api/RuleConfigDeriver.java deleted file mode 100644 index 49cdd281809..00000000000 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/RuleConfigDeriver.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.application.api; - -/** - * Interface to hide dependency on prelude from application package module due to semantic rules - * rewriting. - * - * @author lulf - * @since 5.22 - */ -// TODO: This is not used any more. Do a phased removal while keeping config model compatibility -public interface RuleConfigDeriver { - void derive(String ruleBaseDir, String outputDir) throws Exception; -} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index 3fec8550623..b8b329f2b04 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -53,6 +53,8 @@ public interface Model { * once per deployment. * @param fileDistribution {@link com.yahoo.config.model.api.FileDistribution} instance. */ + // TODO: Remove when 6.206 is the oldest version in use + @Deprecated default void reloadDeployFileDistributor(FileDistribution fileDistribution) { } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a69835626ea..7b28edbb2fc 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -46,7 +46,7 @@ public interface ModelContext { boolean hostedVespa(); Zone zone(); Set<Rotation> rotations(); - default boolean disableFileDistributor() { return false; } + default boolean disableFileDistributor() { return true; } // TODO: Remove when oldest version in use is 6.206 } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 67724058e64..bd94f67e4a7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -68,7 +68,6 @@ public class DeployState implements ConfigDefinitionStore { private final Version wantedNodeVespaVersion; private final Instant now; private final HostProvisioner provisioner; - private final boolean disableFiledistributor; public static DeployState createTestState() { return new Builder().build(); @@ -82,7 +81,7 @@ public class DeployState implements ConfigDefinitionStore { FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, - SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion, boolean disableFiledistributor) { + SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -101,7 +100,6 @@ public class DeployState implements ConfigDefinitionStore { this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; - this.disableFiledistributor = disableFiledistributor; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -215,8 +213,6 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } - public boolean disableFiledistributor() { return disableFiledistributor; } - public static class Builder { private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); @@ -231,7 +227,6 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; - private boolean disableFiledistributor = false; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -293,11 +288,6 @@ public class DeployState implements ConfigDefinitionStore { return this; } - public Builder disableFiledistributor(boolean disableFiledistributor) { - this.disableFiledistributor = disableFiledistributor; - return this; - } - public DeployState build() { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); @@ -305,7 +295,7 @@ public class DeployState implements ConfigDefinitionStore { SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, - zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); + zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 1de7ce62df7..2bfc7a418de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -29,9 +29,7 @@ public class ExactMatch extends Processor { public void process() { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); - if (matching.equals(Matching.Type.EXACT) || - matching.equals(Matching.Type.WORD)) - { + if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { implementExactMatch(field, search); } else if (field.getMatching().getExactMatchTerminator() != null) { warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index a4d7b1b4054..02655906f65 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -25,7 +25,7 @@ import java.util.Set; import java.util.TreeSet; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class TextMatch extends Processor { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 47a644a39d4..a4130c8052c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -412,11 +412,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } @Override - public void reloadDeployFileDistributor(FileDistribution fileDistribution) { - getFileDistributor().reloadDeployFileDistributor(fileDistribution); - } - - @Override public AllocatedHosts allocatedHosts() { return allocatedHosts; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 7a1fab8dbd0..74512e70ebe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -138,8 +138,7 @@ public class VespaModelFactory implements ModelFactory { .rotations(modelContext.properties().rotations()) .zone(zone) .now(clock.instant()) - .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) - .disableFiledistributor(modelContext.properties().disableFileDistributor()); + .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()); modelContext.previousModel().ifPresent(builder::previousModel); return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java deleted file mode 100644 index 22855709208..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin; - -import com.yahoo.binaryprefix.BinaryPrefix; -import com.yahoo.binaryprefix.BinaryScaledAmount; -import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; - -/** - * Options for controlling the behavior of the file distribution services. - * - * @author tonytv - */ -public class FileDistributionOptions implements FiledistributorConfig.Producer { - - private FileDistributionOptions() { - } - - private BinaryScaledAmount uploadBitRate = new BinaryScaledAmount(); - private BinaryScaledAmount downloadBitRate = new BinaryScaledAmount(); - private boolean disableFiledistributor = false; - - - public void downloadBitRate(BinaryScaledAmount amount) { - ensureNonNegative(amount); - downloadBitRate = amount; - } - - public void uploadBitRate(BinaryScaledAmount amount) { - ensureNonNegative(amount); - uploadBitRate = amount; - } - - public void disableFiledistributor(boolean value) { - disableFiledistributor = value; - } - - public boolean disableFiledistributor() { - return disableFiledistributor; - } - - private void ensureNonNegative(BinaryScaledAmount amount) { - if (amount.amount < 0) - throw new IllegalArgumentException("Expected non-negative number, got " + amount.amount); - } - - private int byteRate(BinaryScaledAmount bitRate) { - BinaryScaledAmount byteRate = bitRate.divide(8); - return (int) byteRate.as(BinaryPrefix.unit); - } - - @Override - public void getConfig(FiledistributorConfig.Builder builder) { - builder.maxuploadspeed((double) byteRate(uploadBitRate)); - builder.maxdownloadspeed((double) byteRate(downloadBitRate)); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index 0368f7eaf3e..640a85d9b50 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -5,14 +5,13 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.HostName; import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl; import com.yahoo.vespa.model.container.component.SimpleComponent; /** * @author mortent */ public class IdentityProvider extends SimpleComponent implements IdentityConfig.Producer { - public static final String CLASS = AthenzIdentityProviderImpl.class.getName(); + public static final String CLASS = "com.yahoo.vespa.hosted.athenz.identityprovider.AthenzIdentityProviderImpl"; private final AthenzDomain domain; private final AthenzService service; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index bce78017bdd..62828b314d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -144,9 +144,6 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.loadBalancerAddress().isPresent()) { builder.loadBalancerAddress(options.loadBalancerAddress().get()); } - if (options.disableFiledistributor().isPresent()) { - builder.disableFiledistributor(options.disableFiledistributor().get()); - } } private String[] getConfigModelPluginDirs() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index 866bae6666a..0ebd3987ba4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -44,5 +44,4 @@ public interface CloudConfigOptions { Optional<String> dockerRegistry(); Optional<String> dockerVespaBaseImage(); Optional<String> loadBalancerAddress(); - Optional<Boolean> disableFiledistributor(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 81fc464327e..ac49ec53cbf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -85,7 +85,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { /** * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE */ - static final String HOSTED_VESPA_STATUS_FILE_YINST_SETTING = "cloudconfig_server__tenant_vip_status_file"; + static final String HOSTED_VESPA_STATUS_FILE_INSTALL_SETTING = "cloudconfig_server__tenant_vip_status_file"; public enum Networking { disable, enable } @@ -242,7 +242,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { if (configModelContext.getDeployState().isHosted()) { String name = "status.html"; - Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_YINST_SETTING)); + Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_INSTALL_SETTING)); cluster.addComponent( new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), "http://*/" + name, "https://*/" + name)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java index ffba56fa17f..34e242400d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProvider.java @@ -24,9 +24,7 @@ public class FileDistributionConfigProvider { } public void getConfig(FiledistributorrpcConfig.Builder builder) { - // If disabled config proxy should act as file distributor, so use config proxy port - int port = ConfigProxy.BASEPORT; - builder.connectionspec("tcp/" + host.getHostname() + ":" + port); + builder.connectionspec("tcp/" + host.getHostname() + ":" + ConfigProxy.BASEPORT); } public void getConfig(FilereferencesConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index ad27d86fb84..abd4b604a6d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -133,10 +133,4 @@ public class FileDistributor { dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames()); } - // should only be called during deploy, and only once, since it leads to file distributor - // rescanning all files, which is very expensive ATM (April 2016) - public void reloadDeployFileDistributor(FileDistribution dbHandler) { - dbHandler.reloadDeployFileDistributor(); - } - } diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index 9f8277998f2..cb198c170a5 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -142,55 +142,51 @@ create_cloudconfig_dir(); $session_id_file = "$cloudconfig_dir/$tenant/deploy-session-id"; my $command = shift; +$command ||= "help"; -given($command) { - when ("upload") { - my $application_package = shift; - if (!$opt_F) { - if (!$application_package) { - print "Command failed. No application package specified\n"; - usage("upload"); - exit 1; - } - if (!(-e $application_package)) { - print "Command failed. No such directory found: '$application_package'\n"; - exit 1; - } - check_application_directory($application_package); +if ($command eq "upload") { + my $application_package = shift; + if (!$opt_F) { + if (!$application_package) { + print "Command failed. No application package specified\n"; + usage("upload"); + exit 1; } - - do_http_request("upload", $application_package); - } - when ("prepare") { - my $arg = shift; - if ($arg && looks_like_number($arg) && !(-d $arg)) { - do_http_request("prepare", "", $arg); - } elsif ($arg) { - check_application_directory($arg); - do_http_request("upload", $arg); - do_http_request("prepare"); - } else { - do_http_request("prepare"); - } - } - when ("activate") { - my $session_id = shift; - do_http_request("activate", "", $session_id); - } - when ("fetch") { - my $arg = shift; - if ($arg) { - fetch_active_application($arg); - } else { - usage("fetch", $arg); - } + if (!(-e $application_package)) { + print "Command failed. No such directory found: '$application_package'\n"; + exit 1; + } + check_application_directory($application_package); } - when ("help") { + + do_http_request("upload", $application_package); +} elsif ($command eq "prepare") { my $arg = shift; - usage($command, $arg); - } - default { usage($command); } + if ($arg && looks_like_number($arg) && !(-d $arg)) { + do_http_request("prepare", "", $arg); + } elsif ($arg) { + check_application_directory($arg); + do_http_request("upload", $arg); + do_http_request("prepare"); + } else { + do_http_request("prepare"); + } +} elsif ($command eq "activate") { + my $session_id = shift; + do_http_request("activate", "", $session_id); +} elsif ($command eq "fetch") { + my $arg = shift; + if ($arg) { + fetch_active_application($arg); + } else { + usage("fetch", $arg); + } +} elsif ($command eq "help") { + my $arg = shift; + usage($command, $arg); +} else { + usage($command); } @@ -213,18 +209,21 @@ sub usage { if ($command && $command eq "help") { $command = $arg; } - - given($command) { - when ("upload") { usage_upload(); } - when ("prepare") { usage_prepare(); } - when ("activate") { usage_activate(); } - when ("fetch") { usage_fetch(); } - default { - print "Usage: vespa-deploy [-h] [-v] [-f] [-t] [-p] [<command>] [args]\n"; - print "Supported commands: 'upload', 'prepare', 'activate', 'fetch' and 'help'\n"; - print "Supported options: '-h' (help), '-v' (verbose), '-f' (force/ignore validation errors), '-t' (timeout in seconds), '-p' (config server http port)\n"; - print "Try 'vespa-deploy help <command>' to get more help\n"; - } + $command ||= "help"; + + if ($command eq "upload") { + usage_upload(); + } elsif ($command eq "prepare") { + usage_prepare(); + } elsif ($command eq "activate") { + usage_activate(); + } elsif ($command eq "fetch") { + usage_fetch(); + } else { + print "Usage: vespa-deploy [-h] [-v] [-f] [-t] [-p] [<command>] [args]\n"; + print "Supported commands: 'upload', 'prepare', 'activate', 'fetch' and 'help'\n"; + print "Supported options: '-h' (help), '-v' (verbose), '-f' (force/ignore validation errors), '-t' (timeout in seconds), '-p' (config server http port)\n"; + print "Try 'vespa-deploy help <command>' to get more help\n"; } } @@ -340,18 +339,14 @@ sub do_http_request { my $output; my $exitcode = 1; - given($command) { - when ("upload") { - ($exitcode, $output) = http_upload(\@configsources, $configsource_url, $application_package); - } - when ("prepare") { - $output = http_prepare($configsource_url, $supplied_session_id); - $exitcode = $? >> 8; - } - when ("activate") { - $output = http_activate($configsource_url, $supplied_session_id); - $exitcode = $? >> 8; - } + if ($command eq "upload") { + ($exitcode, $output) = http_upload(\@configsources, $configsource_url, $application_package); + } elsif ($command eq "prepare") { + $output = http_prepare($configsource_url, $supplied_session_id); + $exitcode = $? >> 8; + } elsif ($command eq "activate") { + $output = http_activate($configsource_url, $supplied_session_id); + $exitcode = $? >> 8; } my $response; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 01e1a0ba9fa..ad35eff6467 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -306,7 +306,6 @@ public class AdminTestCase { @Test public void testDisableFileDistributorForAllApps() { DeployState state = new DeployState.Builder() - .disableFiledistributor(true) .properties( new DeployProperties.Builder(). zone(new Zone(Environment.dev, RegionName.from("baz"))). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 1784fe0e974..c698e9f5079 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -19,7 +19,6 @@ public class TestOptions implements CloudConfigOptions { private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); private Optional<Integer> numParallelTenantLoaders = Optional.empty(); - private Optional<Boolean> disableFiledistributor = Optional.empty(); @Override public Optional<Integer> rpcPort() { @@ -118,9 +117,6 @@ public class TestOptions implements CloudConfigOptions { @Override public Optional<String> loadBalancerAddress() { return Optional.empty(); } - @Override - public Optional<Boolean> disableFiledistributor() { return disableFiledistributor; } - public TestOptions numParallelTenantLoaders(int numLoaders) { this.numParallelTenantLoaders = Optional.of(numLoaders); return this; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 1373acdf5d2..7acc1fb3069 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -12,7 +12,6 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.text.XML; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.FileDistributionOptions; import com.yahoo.vespa.model.admin.monitoring.DefaultMonitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.content.cluster.ContentCluster; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index d3879f1ab36..96137652e22 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -265,7 +265,7 @@ public class VespaModelTestCase { assertThat(admin.getConfigservers().size(), is(1)); Set<HostInfo> hosts = model.getHosts(); assertThat(hosts.size(), is(1)); - //logd, config proxy, sentinel, config server, slobrok, log server, file distributor + //logd, config proxy, sentinel, config server, slobrok, log server HostInfo host = hosts.iterator().next(); assertThat(host.getServices().size(), is(6)); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index b705336bb5c..8553b07d683 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -18,6 +18,9 @@ public enum NodeType { tenant, /** A config server */ - config + config, + + /** A host of a (docker) config server node */ + confighost } diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index db96fe0eadb..407b3179a88 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -110,7 +110,7 @@ case $1 in echo "Waiting for config proxy to start" fail=true for ((sleepcount=0;$sleepcount<600;sleepcount=$sleepcount+1)) ; do - usleep 100000 + sleep 0.1 if [ -f $P_CONFIG_PROXY ] && kill -0 `cat $P_CONFIG_PROXY` && vespa-ping-configproxy -s $hname 2>/dev/null then echo "config proxy started (runserver pid `cat $P_CONFIG_PROXY`)" diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 65b02d7dc05..e1e7a5a0c18 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -16,8 +16,6 @@ vespa_generate_config(configdefinitions configserver.def) install_config_definition(configserver.def cloud.config.configserver.def) vespa_generate_config(configdefinitions dispatch.def) install_config_definition(dispatch.def vespa.config.search.dispatch.def) -vespa_generate_config(configdefinitions filedistributor.def) -install_config_definition(filedistributor.def cloud.config.filedistribution.filedistributor.def) vespa_generate_config(configdefinitions filereferences.def) install_config_definition(filereferences.def cloud.config.filedistribution.filereferences.def) vespa_generate_config(configdefinitions fleetcontroller.def) diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index aa807c76c91..494dae8b086 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -4,7 +4,8 @@ namespace=cloud.config # Ports rpcport int default=19070 httpport int default=19071 -numthreads int default=16 +# 0 means use the number of CPU cores available +numRpcThreads int default=0 # ZooKeeper zookeeperserver[].hostname string diff --git a/configdefinitions/src/vespa/filedistributor.def b/configdefinitions/src/vespa/filedistributor.def deleted file mode 100644 index 13dd373eac6..00000000000 --- a/configdefinitions/src/vespa/filedistributor.def +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config.filedistribution - -torrentport int -stateport int default = 0 -hostname string -filedbpath string - -maxdownloadspeed double -maxuploadspeed double diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 967cb06a13a..55c08297b43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; @@ -384,7 +385,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } void redeployAllApplications(Deployer deployer) throws InterruptedException { - ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); + ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), + new DaemonThreadFactory("redeploy apps")); tenants.getAllTenants().forEach(tenant -> listApplicationIds(tenant) .forEach(applicationId -> redeployApplication(applicationId, deployer, deploymentExecutor))); deploymentExecutor.shutdown(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 0435c8e59db..82231fbf5d8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -45,6 +45,12 @@ public interface TenantApplications { Transaction deleteApplication(ApplicationId applicationId); /** + * Removes unused applications + * + */ + void removeUnusedApplications(); + + /** * Closes the application repo. Once a repo has been closed, it should not be used again. */ void close(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index c3d13b86591..d6f34650f8f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -18,15 +18,11 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** @@ -40,13 +36,11 @@ import java.util.logging.Logger; public class ZKTenantApplications implements TenantApplications, PathChildrenCacheListener { private static final Logger log = Logger.getLogger(ZKTenantApplications.class.getName()); - private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); private final Curator curator; private final Path applicationsPath; private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKTenantApplications.class.getName())); - private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final TenantName tenant; @@ -60,10 +54,6 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); this.directoryCache.start(); this.directoryCache.addListener(this); - checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeApplications, - checkForRemovedApplicationsInterval.getSeconds(), - checkForRemovedApplicationsInterval.getSeconds(), - TimeUnit.SECONDS); } public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { @@ -130,7 +120,6 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac public void close() { directoryCache.close(); pathChildrenExecutor.shutdown(); - checkForRemovedApplicationsService.shutdown(); } @Override @@ -151,7 +140,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac } // We might have lost events and might need to remove applications (new applications are // not added by listening for events here, they are added when session is added, see RemoteSessionRepo) - removeApplications(); + removeUnusedApplications(); } private void applicationRemoved(ApplicationId applicationId) { @@ -163,7 +152,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac log.log(LogLevel.DEBUG, Tenants.logPre(applicationId) + "Application added: " + applicationId); } - private void removeApplications() { + public void removeUnusedApplications() { ImmutableSet<ApplicationId> activeApplications = ImmutableSet.copyOf(listApplications()); log.log(LogLevel.DEBUG, "Removing stale applications for tenant '" + tenant + "', not removing these active applications: " + activeApplications); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 4502cc7e223..2f05c1a2259 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -141,7 +141,6 @@ public class ModelContextImpl implements ModelContext { private final boolean hostedVespa; private final Zone zone; private final Set<Rotation> rotations; - private final boolean disableFileDistributor; public Properties(ApplicationId applicationId, boolean multitenant, @@ -149,8 +148,7 @@ public class ModelContextImpl implements ModelContext { HostName loadBalancerName, boolean hostedVespa, Zone zone, - Set<Rotation> rotations, - boolean disableFileDistributor) { + Set<Rotation> rotations) { this.applicationId = applicationId; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; @@ -158,7 +156,6 @@ public class ModelContextImpl implements ModelContext { this.hostedVespa = hostedVespa; this.zone = zone; this.rotations = rotations; - this.disableFileDistributor = disableFileDistributor; } @Override @@ -194,8 +191,6 @@ public class ModelContextImpl implements ModelContext { @Override public Set<Rotation> rotations() { return rotations; } - @Override - public boolean disableFileDistributor() { return disableFileDistributor; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index b7567769afd..c39f85ec87f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -57,7 +57,7 @@ public class FileDistributionImpl implements FileDistribution { log.log(LogLevel.DEBUG, "Executing " + request.methodName() + " against " + target.toString()); target.invokeSync(request, timeout); if (request.isError() && request.errorCode() != ErrorCode.CONNECTION) { - log.log(LogLevel.INFO, request.methodName() + " failed: " + request.errorCode() + " (" + request.errorMessage() + ")"); + log.log(LogLevel.DEBUG, request.methodName() + " failed: " + request.errorCode() + " (" + request.errorMessage() + ")"); } target.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java deleted file mode 100644 index 547c4a1bc2a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.filedistribution; - -import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.recipes.CuratorLock; -import com.yahoo.vespa.curator.recipes.CuratorLockException; - -import java.time.Clock; -import java.time.Duration; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Global filedistribution lock to ensure only one configserver may work on filedistribution. - * The implementation uses a combination of a {@link java.util.concurrent.locks.ReentrantLock} and - * a {@link CuratorLock} to ensure both mutual exclusion within the JVM and - * across JVMs via ZooKeeper. - * - * @author lulf - */ -public class FileDistributionLock implements Lock { - private final Lock processLock; - private final CuratorLock curatorLock; - - public FileDistributionLock(Curator curator, String zkPath) { - this.processLock = new ReentrantLock(); - this.curatorLock = new CuratorLock(curator, zkPath); - } - - @Override - public void lock() { - processLock.lock(); - try { - curatorLock.lock(); - } catch (CuratorLockException e) { - processLock.unlock(); - throw e; - } - } - - @Override - public void lockInterruptibly() throws InterruptedException { - throw new UnsupportedOperationException(); - - } - - @Override - public boolean tryLock() { - if (processLock.tryLock()) { - if (curatorLock.tryLock()) { - return true; - } else { - processLock.unlock(); - return false; - } - } else { - return false; - } - } - - @Override - public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException { - TimeoutBudget budget = new TimeoutBudget(Clock.systemUTC(), Duration.ofMillis(unit.toMillis(timeout))); - if (processLock.tryLock(budget.timeLeft().toMillis(), TimeUnit.MILLISECONDS)) { - if (curatorLock.tryLock(budget.timeLeft().toMillis(), TimeUnit.MILLISECONDS)) { - return true; - } else { - processLock.unlock(); - return false; - } - } else { - return false; - } - } - - @Override - public void unlock() { - try { - curatorLock.unlock(); - } finally { - processLock.unlock(); - } - } - - @Override - public Condition newCondition() { - throw new UnsupportedOperationException(); - } -} - diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index dbd8fdda052..c9859cc79d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -107,7 +107,7 @@ public class FileServer { private void serveFile(FileReference reference, Receiver target) { File file = root.getFile(reference); - log.log(LogLevel.DEBUG, "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(LogLevel.DEBUG, () -> "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); boolean success = false; String errorDescription = "OK"; FileReferenceData fileData = FileReferenceDataBlob.empty(reference, file.getName()); @@ -141,7 +141,7 @@ public class FileServer { private void serveFile(String fileReference, Request request, Receiver receiver) { FileApiErrorCodes result; try { - log.log(LogLevel.DEBUG, "Received request for reference '" + fileReference + "'"); + log.log(LogLevel.DEBUG, () -> "Received request for reference '" + fileReference + "'"); result = hasFile(fileReference) ? FileApiErrorCodes.OK : FileApiErrorCodes.NOT_FOUND; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java index dcc590180f2..d518867407a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java @@ -12,7 +12,6 @@ import java.util.Set; */ public class MockFileDBHandler implements FileDistribution { public int sendDeployedFilesCalled = 0; - public int reloadDeployFileDistributorCalled = 0; public int removeDeploymentsThatHaveDifferentApplicationIdCalled = 0; @Override @@ -26,9 +25,7 @@ public class MockFileDBHandler implements FileDistribution { } @Override - public void reloadDeployFileDistributor() { - reloadDeployFileDistributorCalled++; - } + public void reloadDeployFileDistributor() {} @Override public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 731e343532a..daff32198a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -186,8 +186,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { HostName.from(configserverConfig.loadBalancerAddress()), configserverConfig.hostedVespa(), zone, - rotations, - configserverConfig.disableFiledistributor()); + rotations); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 08095c55373..a5f288bf254 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -116,7 +116,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { this.metrics = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); this.hostLivenessTracker = hostLivenessTracker; BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); - executorService = new ThreadPoolExecutor(config.numthreads(), config.numthreads(), + int numberOfRpcThreads = (config.numRpcThreads() == 0) ? Runtime.getRuntime().availableProcessors() : config.numRpcThreads(); + executorService = new ThreadPoolExecutor(numberOfRpcThreads, numberOfRpcThreads, 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); spec = new Spec(null, config.rpcport()); @@ -461,7 +462,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { request.parameters().add(new StringValue(fileData.filename())); request.parameters().add(new StringValue(fileData.type().name())); request.parameters().add(new Int64Value(fileData.size())); - target.invokeSync(request, 600); + invokeRpcIfValidConnection(request); if (request.isError()) { log.warning("Failed delivering meta for reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); @@ -479,7 +480,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { request.parameters().add(new Int32Value(session)); request.parameters().add(new Int32Value(partId)); request.parameters().add(new DataValue(buf)); - target.invokeSync(request, 600); + invokeRpcIfValidConnection(request); if (request.isError()) { throw new IllegalArgumentException("Failed delivering reference '" + ref.value() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); @@ -496,7 +497,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { request.parameters().add(new Int64Value(fileData.xxhash())); request.parameters().add(new Int32Value(status.getCode())); request.parameters().add(new StringValue(status.getDescription())); - target.invokeSync(request, 600); + invokeRpcIfValidConnection(request); if (request.isError()) { throw new IllegalArgumentException("Failed delivering reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); @@ -506,6 +507,14 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } } + + private void invokeRpcIfValidConnection(Request request) { + if (target.isValid()) { + target.invokeSync(request, 600); + } else { + throw new RuntimeException("Connection to " + target + " is invalid", target.getConnectionLostReason()); + } + } } @SuppressWarnings("UnusedDeclaration") diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index fd645c86d1b..bfed526a130 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -102,7 +102,6 @@ public class SessionPreparer { preparation.writeStateZK(); preparation.writeRotZK(); preparation.distribute(); - preparation.reloadDeployFileDistributor(); } log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + " : " + params.getApplicationId()); @@ -155,8 +154,7 @@ public class SessionPreparer { HostName.from(configserverConfig.loadBalancerAddress()), configserverConfig.hostedVespa(), zone, - rotationsSet, - configserverConfig.disableFiledistributor()); + rotationsSet); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -221,13 +219,6 @@ public class SessionPreparer { checkTimeout("distribute files"); } - void reloadDeployFileDistributor() { - if (prepareResult.asList().isEmpty()) return; - PreparedModelsBuilder.PreparedModelResult aModelResult = prepareResult.asList().get(0); - aModelResult.model.reloadDeployFileDistributor(aModelResult.fileDistributionProvider.getFileDistribution()); - checkTimeout("reload all deployed files in file distributor"); - } - ConfigChangeActions result() { return prepareResult.getConfigChangeActions(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 61145c2a138..934ba9b754d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -90,9 +90,8 @@ public class TenantBuilder { * Create a real tenant from the properties given by this builder. * * @return a new {@link Tenant} instance. - * @throws Exception if building fails */ - public Tenant build() throws Exception { + public Tenant build() { createTenantRequestHandler(); createApplicationRepo(); createRemoteSessionFactory(componentRegistry.getClock()); @@ -140,7 +139,7 @@ public class TenantBuilder { private void createSessionCounter() { if (sessionCounter == null) { - sessionCounter = new SessionCounter(componentRegistry.getCurator(), tenant); + sessionCounter = new SessionCounter(componentRegistry.getConfigCurator(), tenant); } } @@ -169,7 +168,7 @@ public class TenantBuilder { } } - private void createRemoteSessionRepo() throws Exception { + private void createRemoteSessionRepo() { if (remoteSessionRepo == null) { remoteSessionRepo = new RemoteSessionRepo(componentRegistry.getCurator(), remoteSessionFactory, @@ -177,6 +176,8 @@ public class TenantBuilder { tenant, applicationRepo, componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant)), + // TODO: Check if we can avoid using one executor service per tenant. Either one for all + // or have a few executors and hash on tenant name to find out which one to use here createSingleThreadedExecutorService(RemoteSessionRepo.class.getName())); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 7e26e7fe8f5..4595c340b64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -19,6 +19,7 @@ import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.zookeeper.KeeperException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -29,6 +30,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -43,7 +46,7 @@ import java.util.logging.Logger; * To create or delete a tenant, the handler calls {@link Tenants#addTenant} and {@link Tenants#deleteTenant} methods. * This will delete shared state from zookeeper, and return, so it does not mean a tenant is immediately deleted. * - * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all configservers, and + * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all config servers, and * shutdown and delete any per-configserver state. * * @author Vegard Havdal @@ -57,7 +60,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); private static final Path vespaPath = Path.fromString("/vespa"); - + private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); private static final Logger log = Logger.getLogger(Tenants.class.getName()); private final Map<TenantName, Tenant> tenants = new LinkedHashMap<>(); @@ -67,6 +70,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen private final MetricUpdater metricUpdater; private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(Tenants.class.getName())); + private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Curator.DirectoryCache directoryCache; @@ -90,10 +94,14 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); directoryCache.start(); directoryCache.addListener(this); - log.log(LogLevel.INFO, "Creating all tenants"); // TODO: Change to debug + log.log(LogLevel.DEBUG, "Creating all tenants"); createTenants(); notifyTenantsLoaded(); - log.log(LogLevel.INFO, "All tenants created"); // TODO: Change to debug + log.log(LogLevel.DEBUG, "All tenants created"); + checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeUnusedApplications, + checkForRemovedApplicationsInterval.getSeconds(), + checkForRemovedApplicationsInterval.getSeconds(), + TimeUnit.SECONDS); } /** @@ -199,6 +207,11 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen return tenants.get(DEFAULT_TENANT); } + + private void removeUnusedApplications() { + getAllTenants().forEach(tenant -> tenant.getApplicationRepo().removeUnusedApplications()); + } + private void notifyNewTenant(Tenant tenant) { for (TenantListener listener : tenantListeners) { listener.onTenantCreate(tenant.getName(), tenant); @@ -328,6 +341,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen public void close() { directoryCache.close(); pathChildrenExecutor.shutdown(); + checkForRemovedApplicationsService.shutdown(); } public boolean checkThatTenantExists(TenantName tenant) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index 8901bb5b115..dc93886e0c0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.log.LogLevel; import com.yahoo.vespa.curator.recipes.CuratorCounter; -import com.yahoo.vespa.curator.Curator; import java.util.ArrayList; import java.util.Collections; @@ -13,8 +12,7 @@ import java.util.List; * A counter that sets its initial value to the number of apps in zookeeper if no counter value is set. Subclass * this to get that behavior. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InitializedCounter { @@ -22,10 +20,10 @@ public class InitializedCounter { protected final CuratorCounter counter; private final String sessionsDirPath; - public InitializedCounter(Curator curator, String counterPath, String sessionsDirPath) { + public InitializedCounter(ConfigCurator configCurator, String counterPath, String sessionsDirPath) { this.sessionsDirPath = sessionsDirPath; - this.counter = new CuratorCounter(curator, counterPath); - initializeCounterValue(getLatestSessionId(ConfigCurator.create(curator), sessionsDirPath)); + this.counter = new CuratorCounter(configCurator.curator(), counterPath); + initializeCounterValue(getLatestSessionId(configCurator, sessionsDirPath)); } private void initializeCounterValue(Long latestSessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java index 4df292dd204..b8f5acdb225 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java @@ -3,18 +3,16 @@ package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.tenant.Tenants; -import com.yahoo.vespa.curator.Curator; /** * A counter keeping track of session ids in an atomic fashion across multiple config servers. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionCounter extends InitializedCounter { - public SessionCounter(Curator curator, TenantName tenantName) { - super(curator, + public SessionCounter(ConfigCurator configCurator, TenantName tenantName) { + super(configCurator, Tenants.getTenantPath(tenantName).append("sessionCounter").getAbsolute(), Tenants.getSessionsPath(tenantName).getAbsolute()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index e4c336a55f1..aed0a6a9750 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -48,8 +48,7 @@ public class ModelContextImplTest { null, false, Zone.defaultZone(), - rotations, - false), + rotations), Optional.empty(), new Version(6), new Version(6)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java index 03bcb4d71e9..28b24f15a60 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryTenantApplications.java @@ -51,6 +51,11 @@ public class MemoryTenantApplications implements TenantApplications { } @Override + public void removeUnusedApplications() { + // do nothing + } + + @Override public void close() { isOpen = false; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java deleted file mode 100644 index 362934b7898..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.filedistribution; - -import com.yahoo.vespa.config.server.TestWithCurator; -import com.yahoo.vespa.curator.recipes.CuratorLockException; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - -/** - * @author lulf - */ -public class FileDistributionLockTest extends TestWithCurator { - - FileDistributionLock lock; - private int value = 0; - - @Before - public void setupLock() { - lock = new FileDistributionLock(curator, "/lock"); - value = 0; - } - - @Test - public void testDistributedLock() throws InterruptedException, TimeoutException, ExecutionException { - ExecutorService executor = Executors.newFixedThreadPool(20); - - List<Future<?>> futureList = new ArrayList<>(); - for (int i = 0; i < 20; i++) { - futureList.add(executor.submit(() -> { - lock.lock(); - value++; - lock.unlock(); - })); - } - - for (Future<?> future : futureList) { - future.get(600, TimeUnit.SECONDS); - } - assertThat(value, is(20)); - } - - @Test - public void testDistributedTryLockFailure() throws InterruptedException { - MockCurator mockCurator = new MockCurator(); - lock = new FileDistributionLock(mockCurator, "/mocklock"); - mockCurator.timeoutOnLock = true; - assertFalse(lock.tryLock(600, TimeUnit.SECONDS)); - mockCurator.timeoutOnLock = false; - // Second time should not be blocking - Thread t = new Thread(() -> { - try { - if (lock.tryLock(6, TimeUnit.SECONDS)) { - value = 1; - lock.unlock(); - } - } catch (InterruptedException e) { - } - }); - assertThat(value, is(0)); - t.start(); - t.join(); - assertThat(value, is(1)); - } - - @Test - public void testDistributedLockExceptionFailure() throws InterruptedException { - MockCurator mockCurator = new MockCurator(); - lock = new FileDistributionLock(mockCurator, "/mocklock"); - mockCurator.throwExceptionOnLock = true; - try { - lock.lock(); - fail("Lock call should not succeed"); - } catch (CuratorLockException e) { - // ignore - } - mockCurator.throwExceptionOnLock = false; - // Second time should not be blocking - Thread t = new Thread(() -> { - try { - lock.lock(); - value = 1; - lock.unlock(); - } catch (Exception e) { - fail("Should not fail"); - } - }); - assertThat(value, is(0)); - t.start(); - t.join(); - assertThat(value, is(1)); - } - - @Test(expected = UnsupportedOperationException.class) - public void testConditionNotSupported() { - lock.newCondition(); - } - - @Test(expected = UnsupportedOperationException.class) - public void testLockInterruptiblyNotSupported() throws InterruptedException { - lock.lockInterruptibly(); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index 12dc584f055..e022b622fb0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -82,7 +82,11 @@ public class TestWithRpc { protected void createAndStartRpcServer(boolean hostedVespa) { ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); - rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), + rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder() + .rpcport(port) + .numRpcThreads(1) + .maxgetconfigclients(1) + .hostedVespa(hostedVespa)), new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, new SuperModelManager( diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index ac68307f42d..2d182f03de7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -50,7 +50,7 @@ public class LocalSessionRepoTest extends TestWithCurator { } clock = new ManualClock(Instant.ofEpochSecond(1)); LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, - new SessionCounter(globalComponentRegistry.getCurator(), tenantName), + new SessionCounter(globalComponentRegistry.getConfigCurator(), tenantName), new MemoryTenantApplications(), tenantFileSystemDirs, new HostRegistry<>(), tenantName); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java index 878339bd703..50c741c494c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java @@ -173,6 +173,11 @@ public class RemoteSessionRepoTest extends TestWithCurator { } @Override + public void removeUnusedApplications() { + // do nothing + } + + @Override public void close() { } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 2069ae48d76..ac16f1f71a7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -126,15 +126,12 @@ public class SessionPreparerTest extends TestWithCurator { new PrepareParams.Builder().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()).build(), Optional.empty(), tenantPath, Instant.now()); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(0)); - assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(0)); } @Test public void require_that_application_is_prepared() throws Exception { preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(2)); - // Should be called only once no matter how many model versions are built - assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(1)); assertTrue(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java index 67932726df1..b444e09f558 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java @@ -11,8 +11,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InitializedCounterTest extends TestWithCurator { @@ -28,7 +27,7 @@ public class InitializedCounterTest extends TestWithCurator { @Test public void requireThatCounterIsInitializedFromNumberOfSessions() { - InitializedCounter counter = new InitializedCounter(curator, "/counter", "/sessions"); + InitializedCounter counter = new InitializedCounter(configCurator, "/counter", "/sessions"); assertThat(counter.counter.get(), is(2l)); } diff --git a/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala b/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala index 994162695d3..dc94d789f7b 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala @@ -7,7 +7,7 @@ import java.util.logging.{Level, Logger} import com.yahoo.config.ConfigInstance import com.yahoo.container.di.ConfigRetriever._ import com.yahoo.container.di.config.Subscriber -import com.yahoo.log.LogLevel +import com.yahoo.log.LogLevel.DEBUG import scala.annotation.tailrec import scala.collection.JavaConverters._ @@ -31,7 +31,7 @@ final class ConfigRetriever(bootstrapKeys: Set[ConfigKeyT], @tailrec final def getConfigs(componentConfigKeys: Set[ConfigKeyT], leastGeneration: Long): ConfigSnapshot = { require(componentConfigKeys intersect bootstrapKeys isEmpty) - log.log(LogLevel.DEBUG, "getConfigs: " + componentConfigKeys) + log.log(DEBUG, "getConfigs: " + componentConfigKeys) setupComponentSubscriber(componentConfigKeys ++ bootstrapKeys) @@ -43,16 +43,26 @@ final class ConfigRetriever(bootstrapKeys: Set[ConfigKeyT], private def getConfigsOptional(leastGeneration: Long): Option[ConfigSnapshot] = { val newestComponentGeneration = componentSubscriber.waitNextGeneration() + log.log(DEBUG, s"getConfigsOptional: new component generation: $newestComponentGeneration") + // leastGeneration is only used to ensure newer generation when the previous generation was invalidated due to an exception if (newestComponentGeneration < leastGeneration) { None } else if (bootstrapSubscriber.generation < newestComponentGeneration) { val newestBootstrapGeneration = bootstrapSubscriber.waitNextGeneration() + log.log(DEBUG, s"getConfigsOptional: new bootstrap generation: ${bootstrapSubscriber.generation}") bootstrapConfigIfChanged() orElse { - if (newestBootstrapGeneration == newestComponentGeneration) componentsConfigIfChanged() - else None + if (newestBootstrapGeneration == newestComponentGeneration){ + log.log(DEBUG, s"Got new components configs with unchanged bootstrap configs.") + componentsConfigIfChanged() + } else { + // This should not be a normal case, and hence a warning to allow investigation. + log.warning(s"Did not get same generation for bootstrap ($newestBootstrapGeneration) and components configs ($newestComponentGeneration).") + None + } } } else { + // bootstrapGen==componentGen (happens only when a new component subscriber returns first config after bootstrap) componentsConfigIfChanged() } } @@ -61,8 +71,7 @@ final class ConfigRetriever(bootstrapKeys: Set[ConfigKeyT], private def componentsConfigIfChanged(): Option[ComponentsConfigs] = configIfChanged(componentSubscriber, ComponentsConfigs) private def configIfChanged[T <: ConfigSnapshot](subscriber: Subscriber, - constructor: Map[ConfigKeyT, ConfigInstance] => T ): - Option[T] = { + constructor: Map[ConfigKeyT, ConfigInstance] => T ): Option[T] = { if (subscriber.configChanged) Some(constructor(subscriber.config.asScala.toMap)) else None } @@ -77,9 +86,9 @@ final class ConfigRetriever(bootstrapKeys: Set[ConfigKeyT], private def setupComponentSubscriber(keys: Set[ConfigKeyT]) { if (componentSubscriberKeys != keys) { componentSubscriber.close() - componentSubscriberKeys = keys try { + log.log(DEBUG, s"Setting up new component subscriber for keys: $keys") componentSubscriber = subscribe(keys) } catch { case e: Throwable => diff --git a/container-di/src/main/scala/com/yahoo/container/di/Container.scala b/container-di/src/main/scala/com/yahoo/container/di/Container.scala index c6812c52242..50ebff6ece9 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/Container.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/Container.scala @@ -14,6 +14,7 @@ import com.yahoo.container.di.componentgraph.core.ComponentNode.ComponentConstru import com.yahoo.container.di.componentgraph.core.{ComponentGraph, ComponentNode, JerseyNode} import com.yahoo.container.di.config.{RestApiContext, SubscriberFactory} import com.yahoo.container.{BundlesConfig, ComponentsConfig} +import com.yahoo.log.LogLevel.DEBUG import com.yahoo.protect.Process import com.yahoo.vespa.config.ConfigKey @@ -116,22 +117,42 @@ class Container( fallbackInjector: Injector): ComponentGraph = { val snapshot = configurer.getConfigs(graph.configKeys, leastGeneration) - log.fine("""createNewGraph: - graph.configKeys = %s - graph.generation = %s - snapshot = %s - """.format(graph.configKeys, graph.generation, snapshot)) + log.log(DEBUG, + """createNewGraph: + |graph.configKeys = %s + |graph.generation = %s + |snapshot = %s""" + .format(graph.configKeys, graph.generation, snapshot).stripMargin) val preventTailRecursion = snapshot match { case BootstrapConfigs(configs) if getBootstrapGeneration > previousConfigGeneration => + log.log(DEBUG, + """Got new bootstrap generation + |bootstrap generation = %d + |components generation: %d + |previous generation: %d""" + .format(getBootstrapGeneration, getComponentsGeneration, previousConfigGeneration).stripMargin) installBundles(configs) createNewGraph( createComponentsGraph(configs, getBootstrapGeneration,fallbackInjector), fallbackInjector) - case BootstrapConfigs(_) => + case BootstrapConfigs(_) => + // This is an assumed dead code branch, most likely remains from before config set subscriptions were available. + log.warning( + """Got bootstrap configs with previous generation. (This should not happen.) + |bootstrap generation = %d + |components generation: %d + |previous generation: %d""" + .format(getBootstrapGeneration, getComponentsGeneration, previousConfigGeneration).stripMargin) createNewGraph(graph, fallbackInjector) case ComponentsConfigs(configs) => + log.log(DEBUG, + """Got components configs, + |bootstrap generation = %d + |components generation: %d + |previous generation: %d""" + .format(getBootstrapGeneration, getComponentsGeneration, previousConfigGeneration).stripMargin) createAndConfigureComponentsGraph(configs, fallbackInjector) } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/Ckms.java b/container-disc/src/main/java/com/yahoo/container/jdisc/Ckms.java new file mode 100644 index 00000000000..26c71686a82 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/Ckms.java @@ -0,0 +1,14 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.container.jdisc; + +/** + * @author mortent + */ +public interface Ckms { + /** Returns the secret for this key */ + String getSecret(String key); + + /** Returns the secret for this key and version */ + String getSecret(String key, int version); +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index c4c57f4bc47..b7190927d11 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -7,8 +7,7 @@ import javax.net.ssl.SSLContext; * @author mortent */ public interface AthenzIdentityProvider { - String getNToken() throws AthenzIdentityProviderException; String getDomain(); String getService(); - SSLContext getSslContext(); + SSLContext getIdentitySslContext(); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index b83dd6175e1..a986fbc794b 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -40,10 +40,14 @@ public class Deconstructor implements ComponentDeconstructor { executor.schedule(new DestructComponentTask(abstractComponent), delay, TimeUnit.SECONDS); } } else if (component instanceof Provider) { + log.info("Starting deconstruction of " + component); ((Provider)component).deconstruct(); + log.info("Finished deconstructing " + component); } else if (component instanceof SharedResource) { // No need to delay release, as jdisc does ref-counting + log.info("Starting deconstruction of " + component); ((SharedResource)component).release(); + log.info("Finished deconstructing " + component); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/config/RuleConfigDeriver.java b/container-search/src/main/java/com/yahoo/prelude/semantics/config/RuleConfigDeriver.java deleted file mode 100644 index b9fd6fbd51d..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/config/RuleConfigDeriver.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.semantics.config; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.io.Writer; -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.prelude.semantics.RuleBase; -import com.yahoo.prelude.semantics.RuleImporter; -import com.yahoo.prelude.semantics.parser.ParseException; - -/** - * Reads the rule base files in the given directory and creates a - * semantic-rules.cfg file containing those rule bases in the given output dir. - * - * @author bratseth - */ -// Note: This is not used by the config model any more and can be removed -public class RuleConfigDeriver { - - public void derive(String ruleBaseDir, String outputDir) throws IOException, ParseException { - // Validate output dir - File outputDirFile=new File(outputDir); - if (!outputDirFile.exists()) - throw new IOException("Output dir " + outputDirFile.getAbsolutePath() + - " does not exist"); - - List<RuleBase> ruleBases = derive(ruleBaseDir); - // Convert file to config - exportConfig(ruleBases,outputDir); - } - - public List<RuleBase> derive(String ruleBaseDir) throws IOException, ParseException { - // Validate the rule bases - boolean ignoreAutomatas=true; // Don't fail if they are not available in config - List<RuleBase> ruleBases = new RuleImporter(ignoreAutomatas).importDir(ruleBaseDir); - ensureZeroOrOneDefault(ruleBases); - return ruleBases; - } - - public List<RuleBase> derive(List<NamedReader> readers) throws IOException, ParseException { - // Validate the rule bases - boolean ignoreAutomatas = true; // Don't fail if they are not available in config - List<RuleBase> ruleBases = new ArrayList<>(); - RuleImporter importer = new RuleImporter(ignoreAutomatas); - for (NamedReader reader : readers) { - ruleBases.add(importer.importFromReader(reader, reader.getName(), null)); - } - ensureZeroOrOneDefault(ruleBases); - return ruleBases; - } - - private void ensureZeroOrOneDefault(List<RuleBase> ruleBases) throws ParseException { - String defaultName=null; - for (RuleBase ruleBase : ruleBases) { - if (defaultName != null && ruleBase.isDefault()) - throw new ParseException("Both '" + defaultName + "' and '" + ruleBase.getName() + - "' is marked as default, there can only be one"); - if (ruleBase.isDefault()) - defaultName = ruleBase.getName(); - } - } - - private void exportConfig(List<RuleBase> ruleBases, String outputDir) - throws IOException { - BufferedWriter writer=null; - try { - writer=IOUtils.createWriter(outputDir + "/semantic-rules.cfg","utf-8",false); - writer.write("rulebase[" + ruleBases.size() + "]\n"); - for (int i=0; i<ruleBases.size(); i++) { - RuleBase ruleBase= ruleBases.get(i); - writer.write("rulebase[" + i + "].name \"" + ruleBase.getName() + "\"\n"); - writer.write("rulebase[" + i + "].rules \""); - writeRuleBaseAsLine(ruleBase.getSource(),writer); - writer.write("\"\n"); - } - } - finally { - IOUtils.closeWriter(writer); - } - } - - private void writeRuleBaseAsLine(String file, Writer writer) throws IOException { - BufferedReader reader=null; - try { - reader=IOUtils.createReader(file,"utf-8"); - String line; - while (null!=(line=reader.readLine())) { - writer.write(line); - writer.write("\\n"); - } - } - finally { - IOUtils.closeReader(reader); - } - } - - public static void main(String[] args) { - if(args.length<2){ - System.out.println("USAGE: RuleConfigDeriver ruleBaseDir outputDir"); - System.exit(1); - } - - try { - new RuleConfigDeriver().derive(args[0],args[1]); - } - catch (Exception e) { - System.out.println("ERROR: " + collectMessage(e)); - System.exit(1); - } - } - - private static String collectMessage(Throwable e) { - if (e.getCause()==null) - return messageOrName(e); - else - return messageOrName(e) + ": " + collectMessage(e.getCause()); - } - - private static String messageOrName(Throwable e) { - if (e.getMessage()!=null) - return e.getMessage(); - else - return e.getClass().getName(); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java deleted file mode 100644 index 49cb8017074..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.semantics.config.test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.prelude.semantics.config.RuleConfigDeriver; -import com.yahoo.prelude.semantics.parser.ParseException; - -/** - * Tests the rule config deriver by reusing the files in ../test/inheritingrules - * - * @author bratseth - */ -public class RuleConfigDeriverTestCase extends junit.framework.TestCase { - - private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; - - public RuleConfigDeriverTestCase(String name) { - super(name); - } - - public void testRuleConfig() throws IOException, ParseException { - new File("temp/ruleconfigderiver/").mkdirs(); - new RuleConfigDeriver().derive(root + "inheritingrules/","temp/ruleconfigderiver"); - assertEqualFiles(root + "semantic-rules.cfg","temp/ruleconfigderiver/semantic-rules.cfg"); - } - - public void testRuleConfigFromReader() throws IOException, ParseException { - FileReader reader = new FileReader(new File(root) + "/numbers.sr"); - NamedReader namedReader = new NamedReader("numbers", reader); - List<NamedReader> readers = new ArrayList<>(); - readers.add(namedReader); - RuleConfigDeriver deriver = new RuleConfigDeriver(); - deriver.derive(readers); - } - - protected void assertEqualFiles(String correctFileName,String checkFileName) - throws java.io.IOException { - BufferedReader correct=null; - BufferedReader check=null; - try { - correct=IOUtils.createReader(correctFileName); - check = IOUtils.createReader(checkFileName); - String correctLine; - int lineNumber=1; - while ( null != (correctLine=correct.readLine())) { - String checkLine=check.readLine(); - assertNotNull("Too few lines, in " + checkFileName + - ", first missing is\n" + lineNumber + - ": " + correctLine,checkLine); - assertTrue("\nIn " + checkFileName + ":\n" + - "Expected line " + lineNumber + ":\n" + - correctLine.replaceAll("\\\\n","\n") + - "\nGot line " + lineNumber + ":\n" + - checkLine.replaceAll("\\\\n","\n") + "\n", - correctLine.trim().equals(checkLine.trim())); - lineNumber++; - } - assertNull("Excess line(s) in " + checkFileName + " starting at " + - lineNumber, - check.readLine()); - - } - finally { - IOUtils.closeReader(correct); - IOUtils.closeReader(check); - } - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java index a8e5db4f952..e8bc16ca271 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java @@ -26,6 +26,8 @@ public interface ZmsClient { boolean hasTenantAdminAccess(AthenzIdentity athenzIdentity, AthenzDomain tenantDomain); + boolean hasHostedOperatorAccess(AthenzIdentity identity); + // Used before vespa tenancy is established for the domain. boolean isDomainAdmin(AthenzIdentity athenzIdentity, AthenzDomain domain); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 9ee83bec26a..c9f2ff6eb49 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -2,21 +2,20 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; -import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; @@ -110,9 +109,8 @@ public class TenantController { if (existingTenantWithDomain.isPresent()) throw new IllegalArgumentException("Could not create " + tenant + ": The Athens domain '" + domain.getName() + "' is already connected to " + existingTenantWithDomain.get()); - ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()); - try { zmsClient.deleteTenant(domain); } catch (ZmsException ignored) { } - zmsClient.createTenant(domain); + athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()) + .createTenant(domain); } db.createTenant(tenant); log.info("Created " + tenant); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java index b72cec562d8..81692e790a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java @@ -16,7 +16,7 @@ import java.security.cert.X509Certificate; import java.util.Optional; import java.util.concurrent.Executor; -import static com.yahoo.vespa.hosted.controller.athenz.filter.SecurityFilterUtils.sendErrorResponse; +import static com.yahoo.vespa.hosted.controller.restapi.filter.SecurityFilterUtils.sendErrorResponse; /** * Authenticates Athenz principal, either through: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java index 80e14ca7f83..1dc46ed81ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java @@ -21,7 +21,7 @@ import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.athenz.filter.SecurityFilterUtils.sendErrorResponse; +import static com.yahoo.vespa.hosted.controller.restapi.filter.SecurityFilterUtils.sendErrorResponse; /** * A variant of the {@link AthenzPrincipalFilter} to be used in combination with a cookie-based diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java index 8b62a93f8d9..67191d4c09d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java @@ -102,6 +102,11 @@ public class ZmsClientImpl implements ZmsClient { return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), identity); } + @Override + public boolean hasHostedOperatorAccess(AthenzIdentity identity) { + return getOrThrow(() -> hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity)); + } + /** * Used when creating tenancies. As there are no tenancy policies at this point, * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java index 0524cf18568..0a360184da9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java @@ -1,13 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.mock; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -17,12 +19,18 @@ import java.util.Set; public class AthenzDbMock { public final Map<AthenzDomain, Domain> domains = new HashMap<>(); + public final List<AthenzIdentity> hostedOperators = new ArrayList<>(); public AthenzDbMock addDomain(Domain domain) { domains.put(domain.name, domain); return this; } + public AthenzDbMock addHostedOperator(AthenzIdentity athenzIdentity) { + hostedOperators.add(athenzIdentity); + return this; + } + public static class Domain { public final AthenzDomain name; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index ba8bfc2405e..3ee2655108a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -68,17 +68,23 @@ public class ZmsClientMock implements ZmsClient { if (application == null) { throw zmsException(400, "Application '%s' not found", applicationName); } - return domain.admins.contains(identity) || application.acl.get(action).contains(identity); + return isHostedOperator(identity) || domain.admins.contains(identity) || application.acl.get(action).contains(identity); } @Override public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", identity, tenantDomain); - return isDomainAdmin(identity, tenantDomain) || + return isHostedOperator(identity) || isDomainAdmin(identity, tenantDomain) || getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(identity); } @Override + public boolean hasHostedOperatorAccess(AthenzIdentity identity) { + log("hasHostedOperatorAccess(identity='%s')", identity); + return isHostedOperator(identity); + } + + @Override public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { log("isDomainAdmin(principal='%s', domain='%s')", identity, domain); return getDomainOrThrow(domain, false).admins.contains(identity); @@ -109,6 +115,10 @@ public class ZmsClientMock implements ZmsClient { return domain; } + private boolean isHostedOperator(AthenzIdentity identity) { + return athenz.hostedOperators.contains(identity); + } + private static ZmsException zmsException(int code, String message, Object... args) { return new ZmsException(code, String.format(message, args)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java index e9db4f9b717..c6781657b8a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java @@ -98,6 +98,10 @@ public class Path { */ public String getRest() { return rest; } + public String asString() { + return pathString; + } + @Override public String toString() { return "path '" + Arrays.stream(elements).collect(Collectors.joining("/")) + "'"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 6fc65253da3..a99b50f0980 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -64,6 +64,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; @@ -194,6 +195,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploy(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -897,6 +899,40 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } + private HttpResponse notifyJobCompletion(String tenant, String applicationName, HttpRequest request) { + try { + controller.applications().notifyJobCompletion(toJobReport(tenant, applicationName, toSlime(request.getData()).get())); + return new MessageResponse("ok"); + } catch (IllegalStateException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + } + + private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) { + Optional<DeploymentJobs.JobError> jobError = Optional.empty(); + if (report.field("jobError").valid()) { + jobError = Optional.of(DeploymentJobs.JobError.valueOf(report.field("jobError").asString())); + } + return new DeploymentJobs.JobReport( + ApplicationId.from(tenantName, applicationName, report.field("instance").asString()), + DeploymentJobs.JobType.fromJobName(report.field("jobName").asString()), + report.field("projectId").asLong(), + report.field("buildNumber").asLong(), + toSourceRevision(report.field("sourceRevision")), + jobError + ); + } + + private static Optional<SourceRevision> toSourceRevision(Inspector object) { + if (!object.field("repository").valid() || + !object.field("branch").valid() || + !object.field("commit").valid()) { + return Optional.empty(); + } + return Optional.of(new SourceRevision(object.field("repository").asString(), object.field("branch").asString(), + object.field("commit").asString())); + } + private Tenant getTenantOrThrow(String tenantName) { return controller.tenants().tenant(new TenantId(tenantName)) .orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java index 283d700c2bd..2deef474f7c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationInstanceAuthorizer.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType; @@ -45,6 +46,7 @@ public class ApplicationInstanceAuthorizer { Tenant tenant, ApplicationName application) { AthenzDomain principalDomain = principal.getDomain(); + if (isHostedOperator(principal.getIdentity())) return; if (!principalDomain.equals(SCREWDRIVER_DOMAIN)) { throw loggedForbiddenException( @@ -112,6 +114,11 @@ public class ApplicationInstanceAuthorizer { return new NotAuthorizedException(formattedMessage); } + private boolean isHostedOperator(AthenzIdentity identity) { + return athenzClientFactory.createZmsClientWithServicePrincipal() + .hasHostedOperatorAccess(identity); + } + private boolean hasDeployAccessToAthenzApplication(AthenzPrincipal principal, AthenzDomain domain, ApplicationName application) { try { return athenzClientFactory.createZmsClientWithServicePrincipal() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java index 9d45b9a6e09..6a268ce8fda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java @@ -95,7 +95,7 @@ public class Authorizer { return new ForbiddenException(formattedMessage); } - private boolean isTenantAdmin(AthenzIdentity identity, Tenant tenant) { + public boolean isTenantAdmin(AthenzIdentity identity, Tenant tenant) { switch (tenant.tenantType()) { case ATHENS: return isAthenzTenantAdmin(identity, tenant.getAthensDomain().get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java new file mode 100644 index 00000000000..fddc2bb6fa1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -0,0 +1,212 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.application.ApplicationInstanceAuthorizer; +import com.yahoo.vespa.hosted.controller.restapi.application.Authorizer; +import com.yahoo.yolean.chain.After; + +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.NotAuthorizedException; +import javax.ws.rs.WebApplicationException; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.HEAD; +import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; +import static com.yahoo.vespa.hosted.controller.restapi.filter.SecurityFilterUtils.sendErrorResponse; + +/** + * A security filter protects all controller apis. + * + * @author bjorncs + */ +@After("com.yahoo.vespa.hosted.controller.athenz.filter.UserAuthWithAthenzPrincipalFilter") +public class ControllerAuthorizationFilter implements SecurityRequestFilter { + + private static final List<Method> WHITELISTED_METHODS = Arrays.asList(GET, OPTIONS, HEAD); + + private final AthenzClientFactory clientFactory; + private final Controller controller; + private final Authorizer authorizer; + private final ApplicationInstanceAuthorizer applicationInstanceAuthorizer; + private final AuthorizationResponseHandler authorizationResponseHandler; + + public interface AuthorizationResponseHandler { + void handle(ResponseHandler responseHandler, DiscFilterRequest request, WebApplicationException verificationException); + } + + @Inject + public ControllerAuthorizationFilter(AthenzClientFactory clientFactory, + Controller controller, + EntityService entityService, + ZoneRegistry zoneRegistry) { + this(clientFactory, controller, entityService, zoneRegistry, new DefaultAuthorizationResponseHandler()); + } + + ControllerAuthorizationFilter(AthenzClientFactory clientFactory, + Controller controller, + EntityService entityService, + ZoneRegistry zoneRegistry, + AuthorizationResponseHandler authorizationResponseHandler) { + this.clientFactory = clientFactory; + this.controller = controller; + this.authorizer = new Authorizer(controller, entityService, clientFactory); + this.applicationInstanceAuthorizer = new ApplicationInstanceAuthorizer(zoneRegistry, clientFactory); + this.authorizationResponseHandler = authorizationResponseHandler; + } + + // NOTE: Be aware of the ordering of the path pattern matching. Semantics may change if the patterns are evaluated + // in different order. + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + Method method = getMethod(request); + if (isWhiteListedMethod(method)) return; + + try { + Path path = new Path(request.getRequestURI()); + AthenzPrincipal principal = getPrincipalOrThrow(request); + if (isWhiteListedOperation(path, method)) { + // no authz check + } else if (isHostedOperatorOperation(path, method)) { + verifyIsHostedOperator(principal); + } else if (isTenantAdminOperation(path, method)) { + verifyIsTenantAdmin(principal, getTenantId(path)); + } else if (isTenantPipelineOperation(path, method)) { + verifyIsTenantPipelineOperator(principal, getTenantId(path), getApplicationName(path)); + } else { + throw new ForbiddenException("No access control is explicitly declared for this api."); + } + } catch (WebApplicationException e) { + authorizationResponseHandler.handle(handler, request, e); + } + } + + private static boolean isWhiteListedMethod(Method method) { + return WHITELISTED_METHODS.contains(method); + } + + private static boolean isWhiteListedOperation(Path path, Method method) { + return path.matches("/screwdriver/v1/jobsToRun") || // TODO EOL'ed API, remove this once api is gone + path.matches("/application/v4/user") && method == PUT || // Create user tenant + path.matches("/application/v4/tenant/{tenant}") && method == POST || // Create tenant + path.matches("/screwdriver/v1/jobreport"); // TODO To be migrated to application/v4 + } + + private static boolean isHostedOperatorOperation(Path path, Method method) { + if (isWhiteListedOperation(path, method)) return false; + return path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") || + path.matches("/controller/v1/{*}") || + path.matches("/provision/v2/{*}") || + path.matches("/screwdriver/v1/trigger/tenant/{*}") || + path.matches("/zone/v2/{*}"); + } + + private static boolean isTenantAdminOperation(Path path, Method method) { + if (isHostedOperatorOperation(path, method)) return false; + return path.matches("/application/v4/tenant/{tenant}") || + path.matches("/application/v4/tenant/{tenant}/migrateTenantToAthens") || + path.matches("/application/v4/tenant/{tenant}/application/{application}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); + } + + private static boolean isTenantPipelineOperation(Path path, Method method) { + if (isTenantAdminOperation(path, method)) return false; + return path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/promote") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/staging/{*}"); + } + + private void verifyIsHostedOperator(AthenzPrincipal principal) { + if (!isHostedOperator(principal.getIdentity())) { + throw new ForbiddenException("Vespa operator role required"); + } + } + + private void verifyIsTenantAdmin(AthenzPrincipal principal, TenantId tenantId) { + if (!isTenantAdmin(principal.getIdentity(), tenantId)) { + throw new ForbiddenException("Tenant admin or Vespa operator role required"); + } + } + + private void verifyIsTenantPipelineOperator(AthenzPrincipal principal, + TenantId tenantId, + ApplicationName applicationName) { + controller.tenants().tenant(tenantId) + .ifPresent(tenant -> applicationInstanceAuthorizer.throwIfUnauthorized(principal, tenant, applicationName)); + } + + private boolean isHostedOperator(AthenzIdentity identity) { + return clientFactory.createZmsClientWithServicePrincipal() + .hasHostedOperatorAccess(identity); + } + + private boolean isTenantAdmin(AthenzIdentity identity, TenantId tenantId) { + return controller.tenants().tenant(tenantId) + .map(tenant -> authorizer.isTenantAdmin(identity, tenant)) + .orElse(false); + } + + private static TenantId getTenantId(Path path) { + if (!path.matches("/application/v4/tenant/{tenant}/{*}")) + throw new InternalServerErrorException("Unable to handle path: " + path.asString()); + return new TenantId(path.get("tenant")); + } + + private static ApplicationName getApplicationName(Path path) { + if (!path.matches("/application/v4/tenant/{tenant}/application/{application}/{*}")) + throw new InternalServerErrorException("Unable to handle path: " + path.asString()); + return ApplicationName.from(path.get("application")); + } + + private static Method getMethod(DiscFilterRequest request) { + return Method.valueOf(request.getMethod().toUpperCase()); + } + + private static AthenzPrincipal getPrincipalOrThrow(DiscFilterRequest request) { + return getPrincipal(request) + .orElseThrow(() -> new NotAuthorizedException("User not authenticated")); + } + + private static Optional<AthenzPrincipal> getPrincipal(DiscFilterRequest request) { + return Optional.ofNullable(request.getUserPrincipal()) + .map(AthenzPrincipal.class::cast); + } + + /** + * Maps {@link WebApplicationException} to http response ({@link Response}. + */ + static class DefaultAuthorizationResponseHandler implements AuthorizationResponseHandler { + @Override + public void handle(ResponseHandler responseHandler, + DiscFilterRequest request, + WebApplicationException exception) { + sendErrorResponse(responseHandler, exception.getResponse().getStatus(), exception.getMessage()); + } + } + + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SecurityFilterUtils.java index 8e193d3848f..7e7cf7a575d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SecurityFilterUtils.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; +package com.yahoo.vespa.hosted.controller.restapi.filter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,12 +12,12 @@ import com.yahoo.jdisc.handler.ResponseHandler; /** * @author bjorncs */ -class SecurityFilterUtils { +public class SecurityFilterUtils { private static final ObjectMapper mapper = new ObjectMapper(); private SecurityFilterUtils() {} - static void sendErrorResponse(ResponseHandler responseHandler, int statusCode, String message) { + public static void sendErrorResponse(ResponseHandler responseHandler, int statusCode, String message) { Response response = new Response(statusCode); response.headers().put("Content-Type", "application/json"); ObjectNode errorMessage = mapper.createObjectNode(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 44f3197643a..14f3e6b2f61 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -147,6 +147,10 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + /** + * @deprecated Method migrated to application v4 - this method will be removed soon. + */ + @Deprecated private HttpResponse notifyJobCompletion(HttpRequest request) { controller.applications().notifyJobCompletion(toJobReport(toSlime(request.getData()).get())); return new StringResponse("ok"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index ccc1798358d..691a5ef223d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -4,17 +4,15 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; -import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; @@ -23,11 +21,14 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; @@ -63,31 +64,33 @@ public final class ControllerTester { private final MemoryNameService nameService; private final RotationsConfig rotationsConfig; private final ArtifactRepositoryMock artifactRepository; + private final EntityService entityService; private Controller controller; public ControllerTester() { this(new MemoryControllerDb(), new AthenzDbMock(), new ManualClock(), new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), defaultRotationsConfig(), - new MemoryNameService(), new ArtifactRepositoryMock()); + new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService()); } public ControllerTester(ManualClock clock) { this(new MemoryControllerDb(), new AthenzDbMock(), clock, new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), defaultRotationsConfig(), - new MemoryNameService(), new ArtifactRepositoryMock()); + new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService()); } public ControllerTester(RotationsConfig rotationsConfig) { this(new MemoryControllerDb(), new AthenzDbMock(), new ManualClock(), new ConfigServerClientMock(), new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), rotationsConfig, new MemoryNameService(), - new ArtifactRepositoryMock()); + new ArtifactRepositoryMock(), new MemoryEntityService()); } private ControllerTester(ControllerDb db, AthenzDbMock athenzDb, ManualClock clock, ConfigServerClientMock configServer, ZoneRegistryMock zoneRegistry, GitHubMock gitHub, CuratorDb curator, RotationsConfig rotationsConfig, - MemoryNameService nameService, ArtifactRepositoryMock artifactRepository) { + MemoryNameService nameService, ArtifactRepositoryMock artifactRepository, + EntityService entityService) { this.db = db; this.athenzDb = athenzDb; this.clock = clock; @@ -98,8 +101,9 @@ public final class ControllerTester { this.nameService = nameService; this.rotationsConfig = rotationsConfig; this.artifactRepository = artifactRepository; + this.entityService = entityService; this.controller = createController(db, curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, - athenzDb, nameService, artifactRepository); + athenzDb, nameService, artifactRepository, entityService); } public Controller controller() { return controller; } @@ -120,10 +124,12 @@ public final class ControllerTester { public ArtifactRepositoryMock artifactRepository() { return artifactRepository; } + public EntityService entityService() { return entityService; } + /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(db, curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb, - nameService, artifactRepository); + nameService, artifactRepository, entityService); } /** Creates the given tenant and application and deploys it */ @@ -233,12 +239,12 @@ public final class ControllerTester { ConfigServerClientMock configServerClientMock, ManualClock clock, GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, AthenzDbMock athensDb, MemoryNameService nameService, - ArtifactRepository artifactRepository) { + ArtifactRepository artifactRepository, EntityService entityService) { Controller controller = new Controller(db, curator, rotationsConfig, gitHubClientMock, - new MemoryEntityService(), + entityService, new MockOrganization(clock), new MemoryGlobalRoutingService(), zoneRegistryMock, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java new file mode 100644 index 00000000000..ff4ceae1c8e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -0,0 +1,192 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter.DefaultAuthorizationResponseHandler; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler; +import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; +import static com.yahoo.jdisc.http.HttpResponse.Status.FORBIDDEN; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author bjorncs + */ +public class ControllerAuthorizationFilterTest { + private static final ObjectMapper mapper = new ObjectMapper(); + + private static final AthenzUser USER = user("john"); + private static final AthenzUser HOSTED_OPERATOR = user("hosted-operator"); + private static final AthenzDomain TENANT_DOMAIN = new AthenzDomain("tenantdomain"); + private static final AthenzService TENANT_ADMIN = new AthenzService(TENANT_DOMAIN, "adminservice"); + private static final AthenzService TENANT_PIPELINE = HostedAthenzIdentities.from(new ScrewdriverId("12345")); + private static final TenantId TENANT = new TenantId("mytenant"); + private static final ApplicationId APPLICATION = new ApplicationId("myapp"); + + @Test + public void white_listed_operations_are_allowed() { + ControllerAuthorizationFilter filter = createFilter(new ControllerTester()); + assertIsAllowed(invokeFilter(filter, createRequest(PUT, "/application/v4/user", USER))); + assertIsAllowed(invokeFilter(filter, createRequest(POST, "/application/v4/tenant/john", USER))); + assertIsAllowed(invokeFilter(filter, createRequest(DELETE, "/screwdriver/v1/jobsToRun", USER))); + assertIsAllowed(invokeFilter(filter, createRequest(DELETE, "/screwdriver/v1/jobreport", USER))); + } + + @Test + public void only_hosted_operator_can_access_operator_apis() { + ControllerTester controllerTester = new ControllerTester(); + controllerTester.athenzDb().hostedOperators.add(HOSTED_OPERATOR); + + ControllerAuthorizationFilter filter = createFilter(controllerTester); + + List<AthenzIdentity> allowed = singletonList(HOSTED_OPERATOR); + List<AthenzIdentity> forbidden = singletonList(USER); + + testApiAccess(PUT, "/application/v4/tenant/mytenant/application/myapp/deploying", + allowed, forbidden, filter); + testApiAccess(POST, "/screwdriver/v1/trigger/tenant/mytenant/application/myapp/", + allowed, forbidden, filter); + testApiAccess(DELETE, "/provision/v2/provision/enqueue", + allowed, forbidden, filter); + } + + @Test + public void only_hosted_operator_or_tenant_admin_can_access_tenant_admin_apis() { + ControllerTester controllerTester = new ControllerTester(); + controllerTester.athenzDb().hostedOperators.add(HOSTED_OPERATOR); + controllerTester.createTenant(TENANT.id(), TENANT_DOMAIN.getName(), null); + controllerTester.athenzDb().domains.get(TENANT_DOMAIN).admins.add(TENANT_ADMIN); + + ControllerAuthorizationFilter filter = createFilter(controllerTester); + + List<AthenzIdentity> allowed = asList(HOSTED_OPERATOR, TENANT_ADMIN); + List<AthenzIdentity> forbidden = singletonList(USER); + + testApiAccess(DELETE, "/application/v4/tenant/mytenant", + allowed, forbidden, filter); + testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/environment/perf/region/myregion/instance/default/deploy", + allowed, forbidden, filter); + testApiAccess(PUT, "/application/v4/tenant/mytenant/application/myapp/environment/prod/region/myregion/instance/default/global-rotation/override", + allowed, forbidden, filter); + } + + @Test + public void only_hosted_operator_and_screwdriver_project_with_deploy_role_can_access_tenant_pipeline_apis() { + ControllerTester controllerTester = new ControllerTester(); + controllerTester.athenzDb().hostedOperators.add(HOSTED_OPERATOR); + controllerTester.createTenant(TENANT.id(), TENANT_DOMAIN.getName(), null); + controllerTester.createApplication(TENANT, APPLICATION.id(), "default", 12345); + AthenzDbMock.Domain domainMock = controllerTester.athenzDb().domains.get(TENANT_DOMAIN); + domainMock.admins.add(TENANT_ADMIN); + domainMock.applications.get(APPLICATION).addRoleMember(ApplicationAction.deploy, TENANT_PIPELINE); + + ControllerAuthorizationFilter filter = createFilter(controllerTester); + + List<AthenzIdentity> allowed = asList(HOSTED_OPERATOR, TENANT_PIPELINE); + List<AthenzIdentity> forbidden = asList(TENANT_ADMIN, USER); + + testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/environment/prod/region/myregion/instance/default/deploy", + allowed, forbidden, filter); + + testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/jobreport", + allowed, forbidden, filter); + + testApiAccess(POST, "/application/v4/tenant/mytenant/application/myapp/promote", + allowed, forbidden, filter); + } + + private static void testApiAccess(Method method, + String path, + List<? extends AthenzIdentity> allowedIdentities, + List<? extends AthenzIdentity> forbiddenIdentities, + ControllerAuthorizationFilter filter) { + allowedIdentities.forEach( + identity -> assertIsAllowed(invokeFilter(filter, createRequest(method, path, identity)))); + forbiddenIdentities.forEach( + identity -> assertIsForbidden(invokeFilter(filter, createRequest(method, path, identity)))); + } + + private static void assertIsAllowed(Optional<AuthorizationResponse> response) { + assertFalse("Expected no response from filter", response.isPresent()); + } + + private static void assertIsForbidden(Optional<AuthorizationResponse> response) { + assertTrue("Expected a response from filter", response.isPresent()); + assertEquals("Invalid status code", response.get().statusCode, FORBIDDEN); + } + + private static ControllerAuthorizationFilter createFilter(ControllerTester controllerTester) { + return new ControllerAuthorizationFilter(new AthenzClientFactoryMock(controllerTester.athenzDb()), + controllerTester.controller(), + controllerTester.entityService(), + controllerTester.zoneRegistry(), + new DefaultAuthorizationResponseHandler()); + } + + private static Optional<AuthorizationResponse> invokeFilter(ControllerAuthorizationFilter filter, + DiscFilterRequest request) { + MockResponseHandler responseHandlerMock = new MockResponseHandler(); + filter.filter(request, responseHandlerMock); + return Optional.ofNullable(responseHandlerMock.getResponse()) + .map(response -> new AuthorizationResponse(response.getStatus(), getErrorMessage(responseHandlerMock))); + } + + private static DiscFilterRequest createRequest(Method method, String path, AthenzIdentity identity) { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getMethod()).thenReturn(method.name()); + when(request.getRequestURI()).thenReturn(path); + when(request.getUserPrincipal()).thenReturn(new AthenzPrincipal(identity)); + return request; + } + + private static String getErrorMessage(MockResponseHandler responseHandler) { + try { + return mapper.readTree(responseHandler.readAll()).get("message").asText(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static AthenzUser user(String name) { + return new AthenzUser(name); + } + + private static class AuthorizationResponse { + final int statusCode; + final String message; + + AuthorizationResponse(int statusCode, String message) { + this.statusCode = statusCode; + this.message = message; + } + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index d680d943f84..679b0114721 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -36,6 +36,7 @@ import static org.junit.Assert.assertTrue; * @author bratseth * @author jvenstad */ +// TODO Move /application/v4/.../jobreport specific testing to ApplicationApiTest public class ScrewdriverApiTest extends ControllerContainerTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/"; @@ -73,7 +74,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { notifyCompletion(app.id(), projectId, JobType.systemTest, Optional.empty()); // Notifying about unknown job fails - tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport", + tester.containerTester().assertResponse(new Request("http://localhost:8080/application/v4/tenant/tenant1/application/application1/jobreport", jsonReport(app.id(), JobType.productionUsEast3, projectId, 1L, Optional.empty()) .getBytes(StandardCharsets.UTF_8), @@ -183,10 +184,10 @@ public class ScrewdriverApiTest extends ControllerContainerTest { } private void notifyCompletion(ApplicationId app, long projectId, JobType jobType, Optional<JobError> error) throws IOException { - assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport", + assertResponse(new Request("http://localhost:8080/application/v4/tenant/tenant1/application/application1/jobreport", jsonReport(app, jobType, projectId, 1L, error).getBytes(StandardCharsets.UTF_8), Request.Method.POST), - 200, "ok"); + 200, "{\"message\":\"ok\"}"); } private static String jsonReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber, diff --git a/dist/vespa.spec b/dist/vespa.spec index 5c1703bbdbe..ff8b292e627 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -99,7 +99,6 @@ Requires: gdb Requires: net-tools %if 0%{?centos} Requires: llvm3.9 -Requires: devtoolset-7-gdb Requires: vespa-boost >= 1.59.0-6 %define _extra_link_directory /usr/lib64/llvm3.9/lib;/opt/vespa-boost/lib;/opt/vespa-cppunit/lib %define _extra_include_directory /usr/include/llvm3.9;/opt/vespa-boost/include;/opt/vespa-cppunit/include diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 00493e3e016..e006d5aca4c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -45,7 +45,7 @@ public interface Docker { Map<String, Object> getBlkioStats(); } - default boolean networkNATed() { + default boolean networkNPTed() { return false; } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 15e88f4f253..a72865e023a 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -139,7 +139,7 @@ public class DockerImpl implements Docker { } @Override - public boolean networkNATed() { + public boolean networkNPTed() { return config.networkNATed(); } diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java index bbd244dd1dc..294ffafa7e7 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -133,6 +133,8 @@ public class AttributeNode implements ExpressionNode { return handler.values; } else if (value instanceof DocumentUpdate) { return Result.INVALID; + } else if (value instanceof DocumentRemove) { + return Result.INVALID; } return Result.FALSE; } diff --git a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java index 242413c8d3e..d12e8d20fa9 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java @@ -43,6 +43,9 @@ public class DocumentNode implements ExpressionNode { doct = ((DocumentPut)op).getDocument().getDataType(); } else if (op instanceof DocumentUpdate) { doct = ((DocumentUpdate)op).getDocumentType(); + } else if (op instanceof DocumentRemove) { + DocumentRemove removeOp = (DocumentRemove)op; + return (removeOp.getId().getDocType().equals(type) ? op : Boolean.FALSE); } else { throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported."); } diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 508bf7f0b18..5286cec4644 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -250,6 +250,21 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { // TODO Fails: assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", upd)); } + public void testDocumentRemove() throws ParseException { + assertEquals(Result.TRUE, evaluate("test", createRemove("id:ns:test::1"))); + assertEquals(Result.FALSE, evaluate("test", createRemove("id:ns:null::1"))); + assertEquals(Result.FALSE, evaluate("test", createRemove("userdoc:test:1234:1"))); + assertEquals(Result.INVALID, evaluate("test.hint", createRemove("id:ns:test::1"))); + assertEquals(Result.FALSE, evaluate("test.hint", createRemove("id:ns:null::1"))); + assertEquals(Result.INVALID, evaluate("test.hint == 0", createRemove("id:ns:test::1"))); + assertEquals(Result.INVALID, evaluate("test.anything", createRemove("id:ns:test::1"))); + assertEquals(Result.INVALID, evaluate("test and test.hint == 0", createRemove("id:ns:test::1"))); + } + + private DocumentRemove createRemove(String docId) { + return new DocumentRemove(new DocumentId(docId)); + } + public void testInvalidLogic() throws ParseException { DocumentPut put = new DocumentPut(manager.getDocumentType("test"), new DocumentId("doc:scheme:")); DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("test"), new DocumentId("doc:scheme:")); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index db7b48cdc3a..410ac539ff4 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -37,6 +37,7 @@ class DocumentSelectParserTest : public CppUnit::TestFixture { CPPUNIT_TEST(testThatComplexFieldValuesHaveCorrectFieldNames); CPPUNIT_TEST(testBodyFieldDetection); CPPUNIT_TEST(testDocumentUpdates); + CPPUNIT_TEST(testDocumentIdsInRemoves); CPPUNIT_TEST(test_syntax_error_reporting); CPPUNIT_TEST(test_operator_precedence); CPPUNIT_TEST(test_token_used_as_ident_preserves_casing); @@ -102,6 +103,7 @@ public: void testDocumentUpdates2(); void testDocumentUpdates3(); void testDocumentUpdates4(); + void testDocumentIdsInRemoves(); void test_syntax_error_reporting(); void test_operator_precedence(); void test_token_used_as_ident_preserves_casing(); @@ -1238,6 +1240,19 @@ void DocumentSelectParserTest::testDocumentUpdates4() PARSEI("-6 % 10 = -6", *_update[0], True); } +void DocumentSelectParserTest::testDocumentIdsInRemoves() +{ + PARSE("testdoctype1", DocumentId("id:ns:testdoctype1::1"), True); + PARSE("testdoctype1", DocumentId("id:ns:null::1"), False); + PARSE("testdoctype1", DocumentId("userdoc:testdoctype1:1234:1"), False); + PARSE("testdoctype1.headerval", DocumentId("id:ns:testdoctype1::1"), Invalid); + // FIXME: Should ideally be False. As long as there always is an AND node with doctype in a selection expression + // we won't end up sending removes using the wrong route. + PARSE("testdoctype1.headerval", DocumentId("id:ns:null::1"), Invalid); + PARSE("testdoctype1.headerval == 0", DocumentId("id:ns:testdoctype1::1"), Invalid); + PARSE("testdoctype1 and testdoctype1.headerval == 0", DocumentId("id:ns:testdoctype1::1"), Invalid); +} + void DocumentSelectParserTest::testUtf8() { createDocs(); diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp index bbaba0ae7cf..4ecb388f60d 100644 --- a/document/src/vespa/document/select/doctype.cpp +++ b/document/src/vespa/document/select/doctype.cpp @@ -41,7 +41,7 @@ DocType::contains(const Context &context) const _doctype))); } if (context._docId != NULL) { - return ResultList(Result::False); + return ResultList(Result::get((context._docId->getDocType() == _doctype))); } const DocumentUpdate &upd(*context._docUpdate); return ResultList(Result::get( diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index e313763fd0e..29a9548579e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -145,6 +145,14 @@ public class DocumentRouteSelectorPolicy case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: return selector.accepts(((UpdateDocumentMessage)msg).getDocumentUpdate()) != Result.FALSE; + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: { + RemoveDocumentMessage removeMsg = (RemoveDocumentMessage)msg; + if (removeMsg.getDocumentId().hasDocType()) { + return selector.accepts(removeMsg.getDocumentRemove()) != Result.FALSE; + } else { + return true; + } + } case DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE: BatchDocumentUpdateMessage bdu = (BatchDocumentUpdateMessage)msg; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index fba2edf5cfd..9db4d1c0d2e 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -356,10 +356,48 @@ public class PolicyTestCase { "route[1].feed \"myfeed\"\n]"; } - private void assertDistribution(PolicyTestFrame frame, String id, String expected) { - frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), - new DocumentId(id))))); - frame.assertSelect(Arrays.asList(expected)); + @Test + public void remove_document_messages_are_sent_to_the_route_handling_the_given_document_type() { + PolicyTestFrame frame = createFrameWithTwoRoutes(); + + frame.setMessage(createRemove("id:ns:testdoc::1")); + frame.assertSelect(Arrays.asList("testdoc-route")); + + frame.setMessage(createRemove("id:ns:other::1")); + frame.assertSelect(Arrays.asList("other-route")); + } + + @Test + public void remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes() { + PolicyTestFrame frame = createFrameWithTwoRoutes(); + + frame.setMessage(createRemove("userdoc:testdoc:1234:1")); + frame.assertSelect(Arrays.asList("testdoc-route", "other-route")); + + frame.setMessage(createRemove("userdoc:other:1234:1")); + frame.assertSelect(Arrays.asList("testdoc-route", "other-route")); + } + + private PolicyTestFrame createFrameWithTwoRoutes() { + PolicyTestFrame result = new PolicyTestFrame(manager); + result.setHop(new HopSpec("test", createDocumentRouteSelectorConfigWithTwoRoutes()) + .addRecipient("testdoc-route").addRecipient("other-route")); + return result; + } + + private String createDocumentRouteSelectorConfigWithTwoRoutes() { + return "[DocumentRouteSelector:raw:" + + "route[2]\n" + + "route[0].name \"testdoc-route\"\n" + + "route[0].selector \"testdoc and testdoc.stringfield != '0'\"\n" + + "route[0].feed \"\"\n" + + "route[1].name \"other-route\"\n" + + "route[1].selector \"other and other.intfield != '0'\"\n" + + "route[1].feed \"\"\n]"; + } + + private RemoveDocumentMessage createRemove(String docId) { + return new RemoveDocumentMessage(new DocumentId(docId)); } @Test diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 58db3079631..cc598aef786 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -63,6 +63,8 @@ public: void testAND(); void testDocumentRouteSelector(); void testDocumentRouteSelectorIgnore(); + void remove_document_messages_are_sent_to_the_route_handling_the_given_document_type(); + void remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes(); void testExternSend(); void testExternMultipleSlobroks(); void testLoadBalancer(); @@ -103,6 +105,8 @@ Test::Main() { testAND(); TEST_FLUSH(); testDocumentRouteSelector(); TEST_FLUSH(); testDocumentRouteSelectorIgnore(); TEST_FLUSH(); + remove_document_messages_are_sent_to_the_route_handling_the_given_document_type(); TEST_FLUSH(); + remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes(); TEST_FLUSH(); testExternSend(); TEST_FLUSH(); testExternMultipleSlobroks(); TEST_FLUSH(); testLoadBalancer(); TEST_FLUSH(); @@ -673,6 +677,62 @@ Test::testDocumentRouteSelectorIgnore() } namespace { + +vespalib::string +createDocumentRouteSelectorConfigWithTwoRoutes() +{ + return "[DocumentRouteSelector:raw:" + "route[2]\n" + "route[0].name \"testdoc-route\"\n" + "route[0].selector \"testdoc and testdoc.stringfield != '0'\"\n" + "route[0].feed \"\"\n" + "route[1].name \"other-route\"\n" + "route[1].selector \"other and other.intfield != '0'\"\n" + "route[1].feed \"\"\n]"; +} + +std::unique_ptr<TestFrame> +createFrameWithTwoRoutes(DocumentTypeRepo::SP repo) +{ + auto result = std::make_unique<TestFrame>(repo); + result->setHop(mbus::HopSpec("test", createDocumentRouteSelectorConfigWithTwoRoutes()) + .addRecipient("testdoc-route").addRecipient("other-route")); + return result; +} + +std::unique_ptr<RemoveDocumentMessage> +makeRemove(vespalib::string docId) +{ + return std::make_unique<RemoveDocumentMessage>(DocumentId(docId)); +} + +} + +void +Test::remove_document_messages_are_sent_to_the_route_handling_the_given_document_type() +{ + auto frame = createFrameWithTwoRoutes(_repo); + + frame->setMessage(makeRemove("id:ns:testdoc::1")); + EXPECT_TRUE(frame->testSelect({"testdoc-route"})); + + frame->setMessage(makeRemove("id:ns:other::1")); + EXPECT_TRUE(frame->testSelect({"other-route"})); +} + +void +Test::remove_document_messages_with_legacy_document_ids_are_sent_to_all_routes() +{ + auto frame = createFrameWithTwoRoutes(_repo); + + frame->setMessage(makeRemove("userdoc:testdoc:1234:1")); + EXPECT_TRUE(frame->testSelect({"testdoc-route", "other-route"})); + + frame->setMessage(makeRemove("userdoc:other:1234:1")); + EXPECT_TRUE(frame->testSelect({"testdoc-route", "other-route"})); +} + +namespace { string getDefaultDistributionConfig( uint16_t redundancy = 2, uint16_t nodeCount = 10, storage::lib::Distribution::DiskDistribution distr diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index 6756f694267..c763723ada1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -15,6 +15,8 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> + LOG_SETUP(".documentrouteselectorpolicy"); using document::select::Result; @@ -129,6 +131,15 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespali case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: return it->second->contains(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) != Result::False; + case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: { + const RemoveDocumentMessage &removeMsg = static_cast<const RemoveDocumentMessage &>(msg); + if (removeMsg.getDocumentId().hasDocType()) { + return it->second->contains(removeMsg.getDocumentId()) != Result::False; + } else { + return true; + } + } + case DocumentProtocol::MESSAGE_MULTIOPERATION: { const MultiOperationMessage& mom = static_cast<const MultiOperationMessage&>(msg); diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 8378af53098..3e0c0a12a4f 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -27,7 +27,6 @@ vespa_define_module( src/tests/tensor/dense_dot_product_function src/tests/tensor/dense_tensor_address_combiner src/tests/tensor/dense_tensor_builder - src/tests/tensor/dense_tensor_function_optimizer src/tests/tensor/dense_xw_product_function src/tests/tensor/vector_from_doubles_function src/tests/tensor/sparse_tensor_builder diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp index 6b11e2e7034..714eb870b3e 100644 --- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -154,156 +154,6 @@ TEST("require that basic addition works") { //----------------------------------------------------------------------------- -struct InnerProduct { - const TensorEngine &engine; - Function function; - TensorSpec a; - TensorSpec b; - TensorSpec expect; - NodeTypes types; - InterpretedFunction interpreted; - ~InnerProduct() {} - InnerProduct(const vespalib::string &expr) - : engine(DefaultTensorEngine::ref()), - function(Function::parse({"a", "b"}, expr)), - a("null"), b("null"), expect("null"), - types(), - interpreted(engine, function, types) {} - InnerProduct(const vespalib::string &expr, - TensorSpec a_in, - TensorSpec b_in, - TensorSpec expect_in) - : engine(DefaultTensorEngine::ref()), - function(Function::parse(expr)), - a(a_in), b(b_in), expect(expect_in), - types(function, {ValueType::from_spec(a.type()), ValueType::from_spec(b.type())}), - interpreted(engine, function, types) {} - void verify_optimized() const { - EXPECT_LESS(interpreted.program_size(), 4u); - InterpretedFunction::Context ctx(interpreted); - Value::UP va = engine.from_spec(a); - Value::UP vb = engine.from_spec(b); - SimpleObjectParams params({*va,*vb}); - const Value &result = interpreted.eval(ctx, params); - EXPECT_EQUAL(engine.to_spec(result), expect); - } - void verify_not_optimized() const { - EXPECT_EQUAL(4u, interpreted.program_size()); - } -}; - -struct UntypedIP : InnerProduct { - UntypedIP(const vespalib::string &expr) : InnerProduct(expr) { - a = TensorSpec("double").add({}, 2.0); - b = TensorSpec("double").add({}, 3.0); - expect = TensorSpec("double").add({}, 6.0); - } -}; - -struct DotProduct : InnerProduct { - DotProduct(const vespalib::string &expr) - : InnerProduct(expr, - TensorSpec("tensor(x[3])") - .add({{"x", 0}}, 5.0) - .add({{"x", 1}}, 3.0) - .add({{"x", 2}}, 2.0), - TensorSpec("tensor(x[3])") - .add({{"x", 0}}, 7.0) - .add({{"x", 1}}, 11.0) - .add({{"x", 2}}, 13.0), - TensorSpec("double") - .add({}, (5.0 * 7.0) + (3.0 * 11.0) + (2.0 * 13.0))) {} -}; - -struct XW : InnerProduct { - XW(const vespalib::string &expr) - : InnerProduct(expr, - TensorSpec("tensor(x[2])") - .add({{"x", 0}}, 1.0) - .add({{"x", 1}}, 2.0), - TensorSpec("tensor(x[2],y[3])") - .add({{"y", 0},{"x", 0}}, 3.0) - .add({{"y", 0},{"x", 1}}, 5.0) - .add({{"y", 1},{"x", 0}}, 7.0) - .add({{"y", 1},{"x", 1}}, 11.0) - .add({{"y", 2},{"x", 0}}, 13.0) - .add({{"y", 2},{"x", 1}}, 17.0), - TensorSpec("tensor(y[3])") - .add({{"y", 0}}, (1.0 * 3.0) + (2.0 * 5.0)) - .add({{"y", 1}}, (1.0 * 7.0) + (2.0 * 11.0)) - .add({{"y", 2}}, (1.0 * 13.0) + (2.0 * 17.0))) {} -}; - -struct MatMul : InnerProduct { - MatMul(const vespalib::string &expr) - : InnerProduct(expr, - TensorSpec("tensor(x[2],y[2])") - .add({{"x", 0},{"y", 0}}, 1.0) - .add({{"x", 0},{"y", 1}}, 2.0) - .add({{"x", 1},{"y", 0}}, 3.0) - .add({{"x", 1},{"y", 1}}, 5.0), - TensorSpec("tensor(y[2],z[2])") - .add({{"y", 0},{"z", 0}}, 7.0) - .add({{"y", 0},{"z", 1}}, 11.0) - .add({{"y", 1},{"z", 0}}, 13.0) - .add({{"y", 1},{"z", 1}}, 17.0), - TensorSpec("tensor(x[2],z[2])") - .add({{"x", 0},{"z", 0}}, (1.0 * 7.0) + (2.0 * 13.0)) - .add({{"x", 0},{"z", 1}}, (1.0 * 11.0) + (2.0 * 17.0)) - .add({{"x", 1},{"z", 0}}, (3.0 * 7.0) + (5.0 * 13.0)) - .add({{"x", 1},{"z", 1}}, (3.0 * 11.0) + (5.0 * 17.0))) {} -}; - -TEST("require that inner product is not optimized for unknown types") { - TEST_DO(UntypedIP("reduce(a*b,sum)").verify_not_optimized()); - TEST_DO(UntypedIP("reduce(join(a,b,f(x,y)(x*y)),sum)").verify_not_optimized()); -} - -TEST("require that dot product works with tensor function") { - TEST_DO(DotProduct("reduce(a*b,sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*y)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(b*a,sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(x*y)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(y*x)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(y*x)),sum)").verify_optimized()); - TEST_DO(DotProduct("reduce(a*b,sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(b*a,sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(y*x)),sum,x)").verify_optimized()); - TEST_DO(DotProduct("reduce(join(b,a,f(x,y)(y*x)),sum,x)").verify_optimized()); -} - -TEST("require that vector matrix multiplication works with tensor function") { - TEST_DO(XW("reduce(a*b,sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(a,b,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(XW("reduce(b*a,sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(b,a,f(x,y)(x*y)),sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(a,b,f(x,y)(y*x)),sum,x)").verify_optimized()); - TEST_DO(XW("reduce(join(b,a,f(x,y)(y*x)),sum,x)").verify_optimized()); -} - -TEST("require that matrix multiplication is not optimized (yet)") { - TEST_DO(MatMul("reduce(a*b,sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(a,b,f(x,y)(x*y)),sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(b*a,sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(b,a,f(x,y)(x*y)),sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(a,b,f(x,y)(y*x)),sum,y)").verify_not_optimized()); - TEST_DO(MatMul("reduce(join(b,a,f(x,y)(y*x)),sum,y)").verify_not_optimized()); -} - -TEST("require that expressions similar to inner product are not optimized") { - TEST_DO(DotProduct("reduce(a*b,prod)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(a*b,max)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(a+b,sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x+y)),sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*x)),sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(y*y)),sum)").verify_not_optimized()); - TEST_DO(DotProduct("reduce(join(a,b,f(x,y)(x*y*1)),sum)").verify_not_optimized()); -} - -//----------------------------------------------------------------------------- - TEST("require that functions with non-compilable lambdas cannot be interpreted") { auto good_map = Function::parse("map(a,f(x)(x+1))"); auto good_join = Function::parse("join(a,b,f(x,y)(x+y))"); diff --git a/eval/src/tests/tensor/dense_tensor_function_optimizer/CMakeLists.txt b/eval/src/tests/tensor/dense_tensor_function_optimizer/CMakeLists.txt deleted file mode 100644 index 3a95ef776d7..00000000000 --- a/eval/src/tests/tensor/dense_tensor_function_optimizer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_dense_tensor_function_optimizer_test_app TEST - SOURCES - dense_tensor_function_optimizer_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_dense_tensor_function_optimizer_test_app COMMAND eval_dense_tensor_function_optimizer_test_app) diff --git a/eval/src/tests/tensor/dense_tensor_function_optimizer/FILES b/eval/src/tests/tensor/dense_tensor_function_optimizer/FILES deleted file mode 100644 index 3c4ec2f1753..00000000000 --- a/eval/src/tests/tensor/dense_tensor_function_optimizer/FILES +++ /dev/null @@ -1 +0,0 @@ -dense_tensor_function_compiler_test.cpp diff --git a/eval/src/tests/tensor/dense_tensor_function_optimizer/dense_tensor_function_optimizer_test.cpp b/eval/src/tests/tensor/dense_tensor_function_optimizer/dense_tensor_function_optimizer_test.cpp deleted file mode 100644 index 269a1e265c5..00000000000 --- a/eval/src/tests/tensor/dense_tensor_function_optimizer/dense_tensor_function_optimizer_test.cpp +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/tensor/dense/dense_dot_product_function.h> -#include <vespa/eval/tensor/dense/dense_xw_product_function.h> -#include <vespa/eval/eval/operation.h> - -using namespace vespalib::eval; -using namespace vespalib::eval::operation; -using namespace vespalib::eval::tensor_function; -using namespace vespalib::tensor; -using vespalib::Stash; - -//----------------------------------------------------------------------------- - -const TensorFunction & -optimizeDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType, - Stash &stash) -{ - const Node &reduceNode = reduce(join(inject(ValueType::from_spec(lhsType), 1, stash), - inject(ValueType::from_spec(rhsType), 3, stash), - Mul::f, stash), - Aggr::SUM, {}, stash); - return DenseDotProductFunction::optimize(reduceNode, stash); -} - -void assertParam(const TensorFunction &node, size_t expect_idx) { - auto inject = as<Inject>(node); - ASSERT_TRUE(inject); - EXPECT_EQUAL(inject->param_idx(), expect_idx); -} - -void -assertOptimizedDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType) -{ - Stash stash; - const TensorFunction &func = optimizeDotProduct(lhsType, rhsType, stash); - const DenseDotProductFunction *dotProduct = as<DenseDotProductFunction>(func); - ASSERT_TRUE(dotProduct); - TEST_DO(assertParam(dotProduct->lhs(), 1)); - TEST_DO(assertParam(dotProduct->rhs(), 3)); -} - -void -assertNotOptimizedDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType) -{ - Stash stash; - const TensorFunction &func = optimizeDotProduct(lhsType, rhsType, stash); - const Reduce *reduce = as<Reduce>(func); - EXPECT_TRUE(reduce); -} - -//----------------------------------------------------------------------------- - -const TensorFunction & -optimizeXWProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType, - const vespalib::string &dim, - Stash &stash) -{ - const Node &reduceNode = reduce(join(inject(ValueType::from_spec(lhsType), 1, stash), - inject(ValueType::from_spec(rhsType), 3, stash), - Mul::f, stash), - Aggr::SUM, {dim}, stash); - return DenseXWProductFunction::optimize(reduceNode, stash); -} - -void -assertOptimizedXWProduct(const vespalib::string &vecTypeStr, - const vespalib::string &matTypeStr, - const vespalib::string &dim) -{ - Stash stash; - const TensorFunction &func = optimizeXWProduct(vecTypeStr, matTypeStr, dim, stash); - const TensorFunction &inv_func = optimizeXWProduct(matTypeStr, vecTypeStr, dim, stash); - const DenseXWProductFunction *xwProduct = as<DenseXWProductFunction>(func); - const DenseXWProductFunction *inv_xwProduct = as<DenseXWProductFunction>(inv_func); - ValueType vecType = ValueType::from_spec(vecTypeStr); - ValueType matType = ValueType::from_spec(matTypeStr); - size_t common_idx = matType.dimension_index(vecType.dimensions()[0].name); - ASSERT_TRUE(xwProduct); - ASSERT_TRUE(inv_xwProduct); - ASSERT_TRUE(common_idx != ValueType::Dimension::npos); - TEST_DO(assertParam(xwProduct->lhs(), 1)); - TEST_DO(assertParam(inv_xwProduct->lhs(), 3)); - TEST_DO(assertParam(xwProduct->rhs(), 3)); - TEST_DO(assertParam(inv_xwProduct->rhs(), 1)); - EXPECT_EQUAL(xwProduct->vectorSize(), vecType.dimensions()[0].size); - EXPECT_EQUAL(inv_xwProduct->vectorSize(), vecType.dimensions()[0].size); - EXPECT_EQUAL(xwProduct->resultSize(), matType.dimensions()[1 - common_idx].size); - EXPECT_EQUAL(inv_xwProduct->resultSize(), matType.dimensions()[1 - common_idx].size); - EXPECT_EQUAL(xwProduct->matrixHasCommonDimensionInnermost(), (common_idx == 1)); - EXPECT_EQUAL(inv_xwProduct->matrixHasCommonDimensionInnermost(), (common_idx == 1)); -} - -void -assertNotOptimizedXWProduct(const vespalib::string &vecType, - const vespalib::string &matType, - const vespalib::string &dim) -{ - Stash stash; - const TensorFunction &func = optimizeXWProduct(vecType, matType, dim, stash); - const TensorFunction &inv_func = optimizeXWProduct(matType, vecType, dim, stash); - const Reduce *reduce = as<Reduce>(func); - const Reduce *inv_reduce = as<Reduce>(inv_func); - EXPECT_TRUE(reduce); - EXPECT_TRUE(inv_reduce); -} - -//----------------------------------------------------------------------------- - -TEST("require that dot product with compatible dimensions is optimized") -{ - TEST_DO(assertOptimizedDotProduct("tensor(x[5])", "tensor(x[5])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[3])", "tensor(x[5])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[5])", "tensor(x[3])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[])", "tensor(x[5])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[5])", "tensor(x[])")); - TEST_DO(assertOptimizedDotProduct("tensor(x[])", "tensor(x[])")); -} - -TEST("require that dot product with incompatible dimensions is NOT optimized") -{ - TEST_DO(assertNotOptimizedDotProduct("tensor(x[5])", "tensor(y[5])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(y[5])", "tensor(x[5])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(y[])", "tensor(x[])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(x[5])", "tensor(x[5],y[7])")); - TEST_DO(assertNotOptimizedDotProduct("tensor(x[5],y[7])", "tensor(x[5],y[7])")); -} - -//----------------------------------------------------------------------------- - -TEST("require that xw products with compatible dimensions are optimized") { - TEST_DO(assertOptimizedXWProduct("tensor(x[3])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertOptimizedXWProduct("tensor(y[4])", "tensor(x[3],y[4])", "y")); -} - -TEST("require that xw products with incompatible dimensions are not optimized") { - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(x[3],y[4])", "y")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(x[],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(x[3],y[])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[2])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[4])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(y[3],z[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(y[3],z[4])", "y")); - TEST_DO(assertNotOptimizedXWProduct("tensor(x[3])", "tensor(y[3],z[4])", "z")); - TEST_DO(assertNotOptimizedXWProduct("tensor(y[4])", "tensor(x[3],y[4])", "x")); - TEST_DO(assertNotOptimizedXWProduct("tensor(y[3])", "tensor(x[3],y[4])", "y")); - TEST_DO(assertNotOptimizedXWProduct("tensor(y[5])", "tensor(x[3],y[4])", "y")); -} - -//----------------------------------------------------------------------------- - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index 0d98176d631..7627e9f04fb 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -72,7 +72,7 @@ class FileAcquirerImpl implements FileAcquirer { private void logWarning() { if (logCount == 0 || System.currentTimeMillis() > nextLogTime ) { - log.warning("Could not connect to the file distributor '" + spec.toString() + "'" + " - " + this + "@" + System.identityHashCode(this)); + log.warning("Could not connect to the config proxy '" + spec.toString() + "'" + " - " + this + "@" + System.identityHashCode(this)); nextLogTime = System.currentTimeMillis() + Math.min(TimeUnit.DAYS.toMillis(1), @@ -152,7 +152,7 @@ class FileAcquirerImpl implements FileAcquirer { if (request.checkReturnTypes("s")) { return new File(request.returnValues().get(0).asString()); } else if (!request.isError()) { - throw new RuntimeException("Invalid answer from file distributor: " + request.returnValues()); + throw new RuntimeException("Invalid answer from config proxy: " + request.returnValues()); } else if (temporaryError(request.errorCode())) { log.log(LogLevel.INFO, "Retrying waitFor: " + request.errorCode() + " -- " + request.errorMessage()); Thread.sleep(1000); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index 27ada7532f5..c4cd2a073c1 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -63,7 +63,7 @@ public class CompressedFileReference { } static void decompress(File inputFile, File outputDir) throws IOException { - log.log(LogLevel.DEBUG, "Decompressing '" + inputFile + "' into '" + outputDir + "'"); + log.log(LogLevel.DEBUG, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); ArchiveInputStream ais = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile))); decompress(ais, outputDir); ais.close(); @@ -77,7 +77,7 @@ public class CompressedFileReference { File outFile = new File(outputFile, entry.getName()); if (entry.isDirectory()) { if (!(outFile.exists() && outFile.isDirectory())) { - log.log(LogLevel.DEBUG, "Creating dir: " + outFile.getAbsolutePath()); + log.log(LogLevel.DEBUG, () -> "Creating dir: " + outFile.getAbsolutePath()); if (!outFile.mkdirs()) { log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); } @@ -111,7 +111,7 @@ public class CompressedFileReference { } private static void writeFileToTar(ArchiveOutputStream taos, File baseDir, File file) throws IOException { - log.log(LogLevel.DEBUG, "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); + log.log(LogLevel.DEBUG, () -> "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); taos.putArchiveEntry(taos.createArchiveEntry(file, baseDir.toPath().relativize(file.toPath()).toString())); ByteStreams.copy(new FileInputStream(file), taos); taos.closeArchiveEntry(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java index e9d2e9f7e8a..f24d76fe8be 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.DoubleArray; import com.yahoo.jrt.Int32Value; @@ -15,6 +16,8 @@ import java.io.File; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,6 +33,8 @@ public class FileDistributionRpcServer { private final Supervisor supervisor; private final FileDownloader downloader; + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + new DaemonThreadFactory("Rpc executor")); public FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { this.supervisor = supervisor; @@ -74,22 +79,7 @@ public class FileDistributionRpcServer { @SuppressWarnings({"UnusedDeclaration"}) public final void getFile(Request req) { req.detach(); - FileReference fileReference = new FileReference(req.parameters().get(0).asString()); - log.log(LogLevel.DEBUG, "getFile() called for file reference '" + fileReference.value() + "'"); - Optional<File> pathToFile = downloader.getFile(fileReference); - try { - if (pathToFile.isPresent()) { - req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); - log.log(LogLevel.DEBUG, "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); - } else { - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); - req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); - } - } catch (Throwable e) { - log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage()); - req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); - } - req.returnRequest(); + rpcDownloadExecutor.execute(() -> downloadFile(req)); } @SuppressWarnings({"UnusedDeclaration"}) @@ -123,4 +113,23 @@ public class FileDistributionRpcServer { req.returnValues().add(new Int32Value(0)); } + private void downloadFile(Request req) { + FileReference fileReference = new FileReference(req.parameters().get(0).asString()); + log.log(LogLevel.DEBUG, () -> "getFile() called for file reference '" + fileReference.value() + "'"); + Optional<File> pathToFile = downloader.getFile(fileReference); + try { + if (pathToFile.isPresent()) { + req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); + log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); + } else { + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); + req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); + } + } catch (Throwable e) { + log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage()); + req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); + } + req.returnRequest(); + } + } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5655d69593c..bed38bdcac6 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -57,7 +57,7 @@ public class FileDownloader { private Future<Optional<File>> getFutureFile(FileReference fileReference) { Objects.requireNonNull(fileReference, "file reference cannot be null"); File directory = new File(downloadDirectory, fileReference.value()); - log.log(LogLevel.DEBUG, "Checking if there is a file in '" + directory.getAbsolutePath() + "' "); + log.log(LogLevel.DEBUG, () -> "Checking if there is a file in '" + directory.getAbsolutePath() + "' "); Optional<File> file = getFileFromFileSystem(fileReference, directory); if (file.isPresent()) { @@ -65,7 +65,7 @@ public class FileDownloader { future.set(file); return future; } else { - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found in " + + log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' not found in " + directory.getAbsolutePath() + ", starting download"); return queueForAsyncDownload(fileReference, timeout); } @@ -75,7 +75,7 @@ public class FileDownloader { public void queueForAsyncDownload(List<FileReference> fileReferences) { fileReferences.forEach(fileReference -> { if (fileReferenceDownloader.isDownloading(fileReference)) { - log.log(LogLevel.DEBUG, "Already downloading '" + fileReference.value() + "'"); + log.log(LogLevel.DEBUG, () -> "Already downloading '" + fileReference.value() + "'"); } else { queueForAsyncDownload(fileReference); } @@ -117,12 +117,12 @@ public class FileDownloader { private synchronized Future<Optional<File>> queueForAsyncDownload(FileReference fileReference, Duration timeout) { Future<Optional<File>> inProgress = fileReferenceDownloader.addDownloadListener(fileReference, () -> getFile(fileReference)); if (inProgress != null) { - log.log(LogLevel.DEBUG, "Already downloading '" + fileReference.value() + "'"); + log.log(LogLevel.DEBUG, () -> "Already downloading '" + fileReference.value() + "'"); return inProgress; } Future<Optional<File>> future = queueForAsyncDownload(fileReference); - log.log(LogLevel.INFO, "Queued '" + fileReference.value() + "' for download with timeout " + timeout); + log.log(LogLevel.DEBUG, () -> "Queued '" + fileReference.value() + "' for download with timeout " + timeout); return future; } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index a580759b03b..83cf4e1ad80 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -113,7 +113,7 @@ public class FileReceiver { // Unpack if necessary if (fileType == FileReferenceData.Type.compressed) { File decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); - log.log(LogLevel.DEBUG, "Archived file, unpacking " + inprogressFile + " to " + decompressedDir); + log.log(LogLevel.DEBUG, () -> "Archived file, unpacking " + inprogressFile + " to " + decompressedDir); CompressedFileReference.decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } else { @@ -123,7 +123,7 @@ public class FileReceiver { log.log(LogLevel.ERROR, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); throw new RuntimeException("Failed creating directory (" + fileReferenceDir.toPath() + "): ", e); } - log.log(LogLevel.DEBUG, "Uncompressed file, moving to " + file.getAbsolutePath()); + log.log(LogLevel.DEBUG, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); moveFileToDestination(inprogressFile, file); } } catch (IOException e) { @@ -202,11 +202,11 @@ public class FileReceiver { // Unpack if necessary if (fileReferenceData.type() == FileReferenceData.Type.compressed) { File decompressedDir = Files.createTempDirectory(tempDownloadedDir.toPath(), "decompressed").toFile(); - log.log(LogLevel.DEBUG, "Compressed file, unpacking " + tempFile + " to " + decompressedDir); + log.log(LogLevel.DEBUG, () -> "Compressed file, unpacking " + tempFile + " to " + decompressedDir); CompressedFileReference.decompress(tempFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } else { - log.log(LogLevel.DEBUG, "Uncompressed file, moving to " + file.getAbsolutePath()); + log.log(LogLevel.DEBUG, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); Files.createDirectories(fileReferenceDir.toPath()); moveFileToDestination(tempFile, file); } @@ -220,11 +220,11 @@ public class FileReceiver { private static void moveFileToDestination(File tempFile, File destination) { try { Files.move(tempFile.toPath(), destination.toPath()); - log.log(LogLevel.DEBUG, "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); + log.log(LogLevel.DEBUG, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. // so it might be written already). Delete temp file in that case, to avoid filling the disk. - log.log(LogLevel.DEBUG, "File '" + destination.getAbsolutePath() + "' already exists, continuing: " + e.getMessage()); + log.log(LogLevel.DEBUG, () -> "File '" + destination.getAbsolutePath() + "' already exists, continuing: " + e.getMessage()); try { Files.delete(tempFile.toPath()); } catch (IOException ioe) { /* ignore failure */} @@ -237,7 +237,7 @@ public class FileReceiver { @SuppressWarnings({"UnusedDeclaration"}) public final void receiveFileMeta(Request req) { - log.log(LogLevel.DEBUG, "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); String fileName = req.parameters().get(1).asString(); String type = req.parameters().get(2).asString(); @@ -263,7 +263,7 @@ public class FileReceiver { @SuppressWarnings({"UnusedDeclaration"}) public final void receiveFilePart(Request req) { - log.log(LogLevel.DEBUG, "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); @@ -278,14 +278,14 @@ public class FileReceiver { retval = 1; } double completeness = (double) session.currentFileSize / (double) session.fileSize; - log.log(LogLevel.DEBUG, String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value())); + log.log(LogLevel.DEBUG, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value())); downloader.setDownloadStatus(reference, completeness); req.returnValues().add(new Int32Value(retval)); } @SuppressWarnings({"UnusedDeclaration"}) public final void receiveFileEof(Request req) { - log.log(LogLevel.DEBUG, "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); long xxhash = req.parameters().get(2).asInt64(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 20ad2e48fe2..92c483fc3b1 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -11,6 +11,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; +import org.apache.commons.compress.archivers.ArchiveEntry; import java.io.File; import java.time.Duration; @@ -36,7 +37,7 @@ public class FileReferenceDownloader { private final static Duration rpcTimeout = Duration.ofSeconds(10); private final ExecutorService downloadExecutor = - Executors.newFixedThreadPool(10, new DaemonThreadFactory("filereference downloader")); + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new DaemonThreadFactory("filereference downloader")); private final ConnectionPool connectionPool; private final Map<FileReference, FileReferenceDownload> downloads = new LinkedHashMap<>(); private final Map<FileReference, Double> downloadStatus = new HashMap<>(); // between 0 and 1 @@ -51,10 +52,6 @@ public class FileReferenceDownloader { private void startDownload(Duration timeout, FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - synchronized (downloads) { - downloads.put(fileReference, fileReferenceDownload); - downloadStatus.put(fileReference, 0.0); - } long end = System.currentTimeMillis() + timeout.toMillis(); boolean downloadStarted = false; while ((System.currentTimeMillis() < end) && !downloadStarted) { @@ -77,7 +74,12 @@ public class FileReferenceDownloader { } void addToDownloadQueue(FileReferenceDownload fileReferenceDownload) { - log.log(LogLevel.DEBUG, "Will download file reference '" + fileReferenceDownload.fileReference().value() + "' with timeout " + downloadTimeout); + FileReference fileReference = fileReferenceDownload.fileReference(); + log.log(LogLevel.DEBUG, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); + synchronized (downloads) { + downloads.put(fileReference, fileReferenceDownload); + downloadStatus.put(fileReference, 0.0); + } downloadExecutor.submit(() -> startDownload(downloadTimeout, fileReferenceDownload)); } @@ -93,7 +95,7 @@ public class FileReferenceDownloader { downloads.remove(fileReference); download.future().set(Optional.of(file)); } else { - log.log(LogLevel.INFO, "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); + log.log(LogLevel.DEBUG, () -> "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); } } } @@ -105,9 +107,9 @@ public class FileReferenceDownloader { execute(request, connection); if (validateResponse(request)) { - log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + request + "\nSpec: " + connection); + log.log(LogLevel.DEBUG, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection); if (request.returnValues().get(0).asInt32() == 0) { - log.log(LogLevel.DEBUG, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress()); + log.log(LogLevel.DEBUG, () -> "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress()); return true; } else { log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress()); @@ -115,10 +117,10 @@ public class FileReferenceDownloader { return false; } } else { - log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() + + log.log(LogLevel.INFO, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() + ", error code: " + request.errorCode()); if (request.isError() && request.errorCode() == ErrorCode.CONNECTION || request.errorCode() == ErrorCode.TIMEOUT) { - log.log(LogLevel.WARNING, "Setting error for connection " + connection.getAddress()); + log.log(LogLevel.INFO, "Mark connection " + connection.getAddress() + " with error"); connectionPool.setError(connection, request.errorCode()); } return false; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index b01bd96e506..e60ba141c9b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class ForEachExpression extends CompositeExpression { diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index 26abdd43815..65fa83598b6 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -3,6 +3,9 @@ package com.yahoo.logserver; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import com.yahoo.io.SelectLoopHook; @@ -20,18 +23,14 @@ import com.yahoo.logserver.handlers.LogHandler; public class LogDispatcher implements LogHandler, SelectLoopHook { private static final Logger log = Logger.getLogger(LogDispatcher.class.getName()); - private final List<LogHandler> handlers = new ArrayList<>(); - private int messageCount = 0; - private boolean hasBeenShutDown = false; - private boolean batchedMode = false; + private final List<LogHandler> handlers = new CopyOnWriteArrayList<>(); + private final AtomicInteger messageCount = new AtomicInteger(0); + private final AtomicBoolean batchedMode = new AtomicBoolean(false); private final int batchSize = 5000; - private List<LogMessage> currentBatchList; - private int roundCount = 0; - @SuppressWarnings("unused") - private int lastRoundCount = 0; + private final AtomicBoolean hasBeenShutDown = new AtomicBoolean(false); + private List<LogMessage> currentBatchList = null; - public LogDispatcher() { - } + public LogDispatcher() { } /** * Dispatches a message to all the LogHandler instances we've @@ -41,47 +40,53 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * @param msg The LogMessage instance we wish to dispatch to the * plugins */ - public synchronized void handle(LogMessage msg) { + public void handle(LogMessage msg) { if (msg == null) { throw new NullPointerException("LogMessage was null"); } - if (batchedMode) { + if (batchedMode.get()) { addToBatch(msg); } else { - for (LogHandler h : handlers) { - h.handle(msg); - } + send(msg); } - messageCount++; + messageCount.incrementAndGet(); } private void addToBatch(LogMessage msg) { - if (currentBatchList == null) { - currentBatchList = new ArrayList<LogMessage>(batchSize); - currentBatchList.add(msg); - return; - } + List<LogMessage> toSend = null; + synchronized (this) { + if (currentBatchList == null) { + currentBatchList = new ArrayList<LogMessage>(batchSize); + currentBatchList.add(msg); + return; + } - currentBatchList.add(msg); + currentBatchList.add(msg); - if (currentBatchList.size() == batchSize) { - flushBatch(); + if (currentBatchList.size() == batchSize) { + toSend = stealBatch(); + } } + flushBatch(toSend); } - private void flushBatch() { - List<LogMessage> todo; - synchronized(this) { - todo = currentBatchList; - currentBatchList = null; + private void send(List<LogMessage> messages) { + for (LogHandler ht : handlers) { + ht.handle(messages); } - if (todo == null) return; + } + private void send(LogMessage message) { for (LogHandler ht : handlers) { - ht.handle(todo); + ht.handle(message); } } + private void flushBatch(List<LogMessage> todo) { + if (todo == null) { return; } + send(todo); + } + public void handle(List<LogMessage> messages) { throw new IllegalStateException("method not supported"); } @@ -94,12 +99,20 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * but lists of same. */ public void setBatchedMode(boolean batchedMode) { - this.batchedMode = batchedMode; + this.batchedMode.set(batchedMode); } - public synchronized void flush() { - if (batchedMode) { - flushBatch(); + private List<LogMessage> stealBatch() { + List<LogMessage> toSend = null; + synchronized (this) { + toSend = currentBatchList; + currentBatchList = null; + } + return toSend; + } + public void flush() { + if (batchedMode.get()) { + flushBatch(stealBatch()); } for (LogHandler h : handlers) { @@ -110,15 +123,15 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { } } - public synchronized void close() { - if (hasBeenShutDown) { + public void close() { + if (hasBeenShutDown.getAndSet(true)) { throw new IllegalStateException("Shutdown already in progress"); } - hasBeenShutDown = true; for (LogHandler ht : handlers) { if (ht instanceof Thread) { log.fine("Stopping " + ht); + // Todo: Very bad, never do.... ((Thread) ht).interrupt(); } } @@ -134,17 +147,18 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * <p> * If the thread is not alive it will be start()'ed. */ - public synchronized void registerLogHandler(LogHandler ht) { - if (hasBeenShutDown) { - throw new IllegalStateException("Tried to register LogHandler on" + - " LogDispatcher which was shut down"); + public void registerLogHandler(LogHandler ht) { + if (hasBeenShutDown.get()) { + throw new IllegalStateException("Tried to register LogHandler on LogDispatcher which was shut down"); } - if (handlers.contains(ht)) { - log.warning("LogHandler was already registered: " + ht); - return; + synchronized (this) { + if (handlers.contains(ht)) { + log.warning("LogHandler was already registered: " + ht); + return; + } + handlers.add(ht); } - handlers.add(ht); if ((ht instanceof Thread) && (! ((Thread) ht).isAlive())) { ((Thread) ht).start(); @@ -166,19 +180,16 @@ public class LogDispatcher implements LogHandler, SelectLoopHook { * * @return Returns the number of messages that we have seen. */ - public synchronized int getMessageCount() { - return messageCount; + public int getMessageCount() { + return messageCount.get(); } /** * Hook which is called when the select loop has finished. */ public void selectLoopHook(boolean before) { - if (batchedMode) { - flushBatch(); + if (batchedMode.get()) { + flushBatch(stealBatch()); } - - lastRoundCount = messageCount - roundCount; - roundCount = messageCount; } } diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 9484525a8ec..034db16b4e4 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -25,7 +25,13 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>node-repository</artifactId> <version>${project.version}</version> </dependency> @@ -157,6 +163,23 @@ </compilerArgs> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>target/node-admin-app/components</outputDirectory> + <includeArtifactIds>bcprov-jdk15on,bcpkix-jdk15on</includeArtifactIds> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/node-admin/scripts/maintenance.sh b/node-admin/scripts/maintenance.sh index 4438d5d684b..4438d5d684b 100644..100755 --- a/node-admin/scripts/maintenance.sh +++ b/node-admin/scripts/maintenance.sh diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index 99733552f76..f2f9e46c5b8 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -11,7 +11,9 @@ <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> - <nodes type="host"/> + <config name="vespa.hosted.dockerapi.docker"> + <uri>unix:///var/run/docker.sock</uri> + </config> <preprocess:include file="variant.xml" required="false"/> </jdisc> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java index 15ade142b5d..89f52a39fbb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/DockerAdminComponent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/DockerAdminComponent.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.nodeadmin; +package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.net.HostName; @@ -7,20 +7,17 @@ import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.component.AdminComponent; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerClients; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; import java.time.Clock; import java.time.Duration; @@ -35,25 +32,42 @@ public class DockerAdminComponent implements AdminComponent { private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); private final ConfigServerConfig configServerConfig; - private final NodeAdminConfig config; private final Docker docker; private final MetricReceiverWrapper metricReceiver; - private final ClassLocking classLocking; - - private ConfigServerHttpRequestExecutor requestExecutor; + private final Optional<ClassLocking> classLocking; + private final ConfigServerClients configServerClients; + private Optional<Environment> environment = Optional.empty(); private Optional<NodeAdminStateUpdaterImpl> nodeAdminStateUpdater = Optional.empty(); public DockerAdminComponent(ConfigServerConfig configServerConfig, - NodeAdminConfig config, Docker docker, MetricReceiverWrapper metricReceiver, - ClassLocking classLocking) { + ClassLocking classLocking, + ConfigServerClients configServerClients) { + this(configServerConfig, docker, metricReceiver, Optional.empty(), Optional.of(classLocking), configServerClients); + } + + public DockerAdminComponent(ConfigServerConfig configServerConfig, + Docker docker, + MetricReceiverWrapper metricReceiver, + Environment environment, + ConfigServerClients configServerClients) { + this(configServerConfig, docker, metricReceiver, Optional.of(environment), Optional.empty(), configServerClients); + } + + private DockerAdminComponent(ConfigServerConfig configServerConfig, + Docker docker, + MetricReceiverWrapper metricReceiver, + Optional<Environment> environment, + Optional<ClassLocking> classLocking, + ConfigServerClients configServerClients) { this.configServerConfig = configServerConfig; - this.config = config; this.docker = docker; this.metricReceiver = metricReceiver; + this.environment = environment; this.classLocking = classLocking; + this.configServerClients = configServerClients; } @Override @@ -62,15 +76,14 @@ public class DockerAdminComponent implements AdminComponent { return; } - Environment environment = new Environment(configServerConfig); - requestExecutor = ConfigServerHttpRequestExecutor.create( - environment.getConfigServerUris(), - environment.getKeyStoreOptions(), - environment.getTrustStoreOptions(), - environment.getAthenzIdentity()); + nodeAdminStateUpdater = Optional.of(createNodeAdminStateUpdater()); + nodeAdminStateUpdater.get().start(); + } - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); + private NodeAdminStateUpdaterImpl createNodeAdminStateUpdater() { + if (!environment.isPresent()) { + environment = Optional.of(new Environment(configServerConfig)); + } Clock clock = Clock.systemUTC(); String dockerHostHostName = HostName.getLocalhost(); @@ -79,29 +92,29 @@ public class DockerAdminComponent implements AdminComponent { docker.start(); DockerOperations dockerOperations = new DockerOperationsImpl( docker, - environment, + environment.get(), processExecuter); StorageMaintainer storageMaintainer = new StorageMaintainer( dockerOperations, processExecuter, metricReceiver, - environment, + environment.get(), clock); AclMaintainer aclMaintainer = new AclMaintainer( dockerOperations, - nodeRepository, + configServerClients.nodeRepository(), dockerHostHostName); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( hostName, - nodeRepository, - orchestrator, + configServerClients.nodeRepository(), + configServerClients.orchestrator(), dockerOperations, storageMaintainer, aclMaintainer, - environment, + environment.get(), clock, NODE_AGENT_SCAN_INTERVAL); @@ -113,17 +126,15 @@ public class DockerAdminComponent implements AdminComponent { metricReceiver, clock); - nodeAdminStateUpdater = Optional.of(new NodeAdminStateUpdaterImpl( - nodeRepository, - orchestrator, + return new NodeAdminStateUpdaterImpl( + configServerClients.nodeRepository(), + configServerClients.orchestrator(), storageMaintainer, nodeAdmin, dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, - classLocking)); - - nodeAdminStateUpdater.get().start(); + classLocking); } @Override @@ -132,10 +143,9 @@ public class DockerAdminComponent implements AdminComponent { return; } - nodeAdminStateUpdater.get().stop(); - requestExecutor.close(); + nodeAdminStateUpdater.ifPresent(NodeAdminStateUpdaterImpl::stop); + configServerClients.stop(); nodeAdminStateUpdater = Optional.empty(); - // TODO: Also stop docker } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index 0415bbc34c2..06663b3f0f5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -1,12 +1,15 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; import com.google.common.base.Strings; +import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; import java.net.InetAddress; import java.net.URI; @@ -39,62 +42,81 @@ public class Environment { private static final String LOGSTASH_NODES = "LOGSTASH_NODES"; private static final String COREDUMP_FEED_ENDPOINT = "COREDUMP_FEED_ENDPOINT"; - private final List<URI> configServerHosts; + private final List<URI> configServerURIs; private final String environment; private final String region; private final String parentHostHostname; private final InetAddressResolver inetAddressResolver; private final PathResolver pathResolver; private final List<String> logstashNodes; - private final String feedEndpoint; + private final Optional<String> feedEndpoint; private final Optional<KeyStoreOptions> keyStoreOptions; private final Optional<KeyStoreOptions> trustStoreOptions; private final Optional<AthenzIdentity> athenzIdentity; + private final NodeType nodeType; static { filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); } public Environment(ConfigServerConfig configServerConfig) { + this(configServerConfig, + getEnvironmentVariable(ENVIRONMENT), + getEnvironmentVariable(REGION), + new PathResolver(), + Optional.of(getEnvironmentVariable(COREDUMP_FEED_ENDPOINT)), + NodeType.host); + } + + public Environment(ConfigServerConfig configServerConfig, + String hostedEnvironment, + String hostedRegion, + PathResolver pathResolver, + Optional<String> coreDumpFeedEndpoint, + NodeType nodeType) { this(createConfigServerUris( configServerConfig.scheme(), configServerConfig.hosts(), configServerConfig.port()), - getEnvironmentVariable(ENVIRONMENT), - getEnvironmentVariable(REGION), - Defaults.getDefaults().vespaHostname(), - new InetAddressResolver(), - new PathResolver(), - getLogstashNodesFromEnvironment(), - getEnvironmentVariable(COREDUMP_FEED_ENDPOINT), - - createKeyStoreOptions( - configServerConfig.keyStoreConfig().path(), - configServerConfig.keyStoreConfig().password().toCharArray(), - configServerConfig.keyStoreConfig().type().name()), - createKeyStoreOptions( - configServerConfig.trustStoreConfig().path(), - configServerConfig.trustStoreConfig().password().toCharArray(), - configServerConfig.trustStoreConfig().type().name()), - createAthenzIdentity( - configServerConfig.athenzDomain(), - configServerConfig.serviceName()) + hostedEnvironment, + hostedRegion, + Defaults.getDefaults().vespaHostname(), + new InetAddressResolver(), + pathResolver, + getLogstashNodesFromEnvironment(), + coreDumpFeedEndpoint, + + createKeyStoreOptions( + configServerConfig.keyStoreConfig().path(), + configServerConfig.keyStoreConfig().password().toCharArray(), + configServerConfig.keyStoreConfig().type().name(), + "BC"), + createKeyStoreOptions( + configServerConfig.trustStoreConfig().path(), + configServerConfig.trustStoreConfig().password().toCharArray(), + configServerConfig.trustStoreConfig().type().name(), + null), + createAthenzIdentity( + configServerConfig.athenzDomain(), + configServerConfig.serviceName()), + nodeType ); } - public Environment(List<URI> configServerHosts, + public Environment(List<URI> configServerURIs, String environment, String region, String parentHostHostname, InetAddressResolver inetAddressResolver, PathResolver pathResolver, List<String> logstashNodes, - String feedEndpoint, + Optional<String> feedEndpoint, Optional<KeyStoreOptions> keyStoreOptions, Optional<KeyStoreOptions> trustStoreOptions, - Optional<AthenzIdentity> athenzIdentity) { - this.configServerHosts = configServerHosts; + Optional<AthenzIdentity> athenzIdentity, + NodeType nodeType) { + this.configServerURIs = configServerURIs; this.environment = environment; this.region = region; this.parentHostHostname = parentHostHostname; @@ -105,9 +127,10 @@ public class Environment { this.keyStoreOptions = keyStoreOptions; this.trustStoreOptions = trustStoreOptions; this.athenzIdentity = athenzIdentity; + this.nodeType = nodeType; } - public List<URI> getConfigServerUris() { return configServerHosts; } + public List<URI> getConfigServerUris() { return configServerURIs; } public String getEnvironment() { return environment; @@ -133,7 +156,7 @@ public class Environment { return getEnvironment() + "." + getRegion(); } - private static List<URI> createConfigServerUris(String scheme, List<String> configServerHosts, int port) { + public static List<URI> createConfigServerUris(String scheme, List<String> configServerHosts, int port) { return configServerHosts.stream() .map(hostname -> URI.create(scheme + "://" + hostname + ":" + port)) .collect(Collectors.toList()); @@ -147,10 +170,10 @@ public class Environment { return Arrays.asList(logstashNodes.split("[,\\s]+")); } - private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type) { + private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type, String provider) { return Optional.ofNullable(pathToKeyStore) .filter(path -> !Strings.isNullOrEmpty(path)) - .map(path -> new KeyStoreOptions(Paths.get(path), password, type)); + .map(path -> new KeyStoreOptions(Paths.get(path), password, type, provider)); } private static Optional<AthenzIdentity> createAthenzIdentity(String athenzDomain, String serviceName) { @@ -166,7 +189,7 @@ public class Environment { return pathResolver; } - public String getCoredumpFeedEndpoint() { + public Optional<String> getCoredumpFeedEndpoint() { return feedEndpoint; } @@ -236,24 +259,24 @@ public class Environment { return athenzIdentity; } + public NodeType getNodeType() { return nodeType; } public static class Builder { - private List<URI> configServerHosts = Collections.emptyList(); + private List<URI> configServerURIs = Collections.emptyList(); private String environment; private String region; private String parentHostHostname; private InetAddressResolver inetAddressResolver; private PathResolver pathResolver; private List<String> logstashNodes = Collections.emptyList(); - private String feedEndpoint; + private Optional<String> feedEndpoint = Optional.empty(); private KeyStoreOptions keyStoreOptions; private KeyStoreOptions trustStoreOptions; private AthenzIdentity athenzIdentity; + private NodeType nodeType = NodeType.tenant; - public Builder configServerUris(String... hosts) { - configServerHosts = Arrays.stream(hosts) - .map(URI::create) - .collect(Collectors.toList()); + public Builder configServerUris(List<URI> uris) { + configServerURIs = uris; return this; } @@ -288,7 +311,7 @@ public class Environment { } public Builder feedEndpoint(String feedEndpoint) { - this.feedEndpoint = feedEndpoint; + this.feedEndpoint = Optional.of(feedEndpoint); return this; } @@ -307,11 +330,17 @@ public class Environment { return this; } + public Builder nodeType(NodeType nodeType) { + this.nodeType = nodeType; + return this; + } + public Environment build() { - return new Environment(configServerHosts, environment, region, parentHostHostname, inetAddressResolver, + return new Environment(configServerURIs, environment, region, parentHostHostname, inetAddressResolver, pathResolver, logstashNodes, feedEndpoint, Optional.ofNullable(keyStoreOptions), Optional.ofNullable(trustStoreOptions), - Optional.ofNullable(athenzIdentity)); + Optional.ofNullable(athenzIdentity), + nodeType); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java index b6b64dbf5dd..d2c09aae22a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java @@ -3,19 +3,39 @@ package com.yahoo.vespa.hosted.node.admin.component; /** * This class is thread unsafe: All method calls MUST be exclusive and serialized. + * + * In a specialized environment it is possible to provide a richer context than TaskContext: + * - Define a subclass T of TaskContext with the additional functionality. + * - Define task classes that implement IdempotentTask<T>. */ -public interface IdempotentTask { - String name(); +public interface IdempotentTask<T extends TaskContext> { + /** + * A short id of the task to e.g. identify the task in the log. + * + * Prefer PascalCase and without white-space. + * + * Example: "EnableDocker" + */ + default String name() { return getClass().getSimpleName(); } /** - * Execute an administrative task to converge the system towards some ideal state. + * Execute an administrative task to converge towards some ideal state, whether it is + * system state or in-memory Java state. * * converge() must be idempotent: it may be called any number of times, or - * interrupted at any time e.g. by `kill -9`. The caller must ensure there is at - * most one invocation of converge() on this instance at any given time. + * interrupted at any time e.g. by `kill -9`. + * + * converge() is not thread safe: The caller must ensure there is at most one invocation + * of converge() at any given time. * - * @return false if the system was already converged, i.e. converge() was a no-op. + * @return false if already converged, i.e. was a no-op. A typical sequence of converge() + * calls on a IdempotentTask will consist of: + * - Any number of calls that throws an exception due to some issues. Assuming + * no exceptions were thrown, or the issue eventually resolved itself... + * (convergence failure) + * - Returns true once (converged just now) + * - Returns false for all further calls (already converged) * @throws RuntimeException (or a subclass) if the task is unable to converge. */ - boolean converge(TaskContext context); + boolean converge(T context); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PathResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java index 2e26ff0773b..f70f451d33d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/PathResolver.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.component; import java.nio.file.Path; import java.nio.file.Paths; @@ -8,9 +8,9 @@ import java.nio.file.Paths; * @author freva */ public class PathResolver { - static final Path ROOT = Paths.get("/"); + public static final Path ROOT = Paths.get("/"); + public static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - private static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); private final Path applicationStoragePathForNodeAdmin; private final Path applicationStoragePathForHost; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java index c54f9ee00c8..cbe9b32cc47 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskComponent.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.ChainedComponent; -public abstract class TaskComponent extends ChainedComponent implements IdempotentTask { +public abstract class TaskComponent extends ChainedComponent implements IdempotentTask<TaskContext> { protected TaskComponent(ComponentId id) { super(id); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index d0a5570b8dc..0c49e478d6a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -50,10 +50,4 @@ public interface TaskContext { * or later if the task failed. Either way, it will only be called once. */ default void logOnFailure(Logger logger, Supplier<String> messageSupplier) {} - - /** - * Execute a task as a child of this task, and with its own sub-TaskContext. Please avoid - * excessive task hierarchies. - */ - boolean executeSubtask(IdempotentTask task); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java index 108d42114f7..bc0ec2c8700 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; import java.util.logging.Logger; public class TestTaskContext implements TaskContext { @@ -18,9 +17,6 @@ public class TestTaskContext implements TaskContext { @Override public void log(Logger logger, String message) { } - @Override - public void logOnFailure(Logger logger, Supplier<String> messageSupplier) { } - public List<String> getSystemModificationLog() { return systemModifications; } @@ -28,9 +24,4 @@ public class TestTaskContext implements TaskContext { public void clearSystemModificationLog() { systemModifications.clear(); } - - @Override - public boolean executeSubtask(IdempotentTask task) { - throw new UnsupportedOperationException(); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java new file mode 100644 index 00000000000..4d4a6c0328d --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java @@ -0,0 +1,28 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import java.util.Optional; + +/** + * Interface to execute basic HTTP request against config server(s) + * + * @author freva + */ +public interface ConfigServerApi extends AutoCloseable { + + <T> T get(String path, Class<T> wantedReturnType); + + <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType); + + <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType); + + <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType); + + <T> T delete(String path, Class<T> wantedReturnType); + + /** + * Close the underlying HTTP client and any threads this class might have started. + */ + @Override + void close(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 13bfc949533..c5592e91973 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +package com.yahoo.vespa.hosted.node.admin.configserver; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; +import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -18,28 +15,15 @@ import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.StringEntity; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import java.io.FileInputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; -import java.nio.file.Path; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.Security; -import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; /** * Retries request on config server a few times before giving up. Assumes that all requests should be sent with @@ -47,13 +31,11 @@ import java.util.function.Supplier; * * @author dybdahl */ -public class ConfigServerHttpRequestExecutor implements AutoCloseable { - private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(ConfigServerHttpRequestExecutor.class); - private static final Duration CLIENT_REFRESH_INTERVAL = Duration.ofHours(1); +public class ConfigServerApiImpl implements ConfigServerApi { + private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(ConfigServerApiImpl.class); private final ObjectMapper mapper = new ObjectMapper(); - private final ScheduledExecutorService clientRefresherScheduler = - Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("http-client-refresher")); + private final List<URI> configServerHosts; /** @@ -66,30 +48,20 @@ public class ConfigServerHttpRequestExecutor implements AutoCloseable { */ private volatile SelfCloseableHttpClient client; - public static ConfigServerHttpRequestExecutor create( - Collection<URI> configServerUris, - Optional<KeyStoreOptions> keyStoreOptions, - Optional<KeyStoreOptions> trustStoreOptions, - Optional<AthenzIdentity> athenzIdentity) { - Security.addProvider(new BouncyCastleProvider()); - - Supplier<SelfCloseableHttpClient> clientSupplier = () -> createHttpClient(keyStoreOptions, trustStoreOptions, athenzIdentity); - ConfigServerHttpRequestExecutor requestExecutor = new ConfigServerHttpRequestExecutor( - randomizeConfigServerUris(configServerUris), clientSupplier.get()); + public ConfigServerApiImpl(Collection<URI> configServerUris) { + this(configServerUris, SSLConnectionSocketFactory.getSocketFactory()); + } - if (keyStoreOptions.isPresent() || trustStoreOptions.isPresent()) { - requestExecutor.clientRefresherScheduler.scheduleAtFixedRate(() -> requestExecutor.client = clientSupplier.get(), - CLIENT_REFRESH_INTERVAL.toMillis(), CLIENT_REFRESH_INTERVAL.toMillis(), TimeUnit.MILLISECONDS); - } - return requestExecutor; + ConfigServerApiImpl(Collection<URI> configServerUris, SSLConnectionSocketFactory sslConnectionSocketFactory) { + this(randomizeConfigServerUris(configServerUris), new SelfCloseableHttpClient(sslConnectionSocketFactory)); } - ConfigServerHttpRequestExecutor(List<URI> configServerHosts, SelfCloseableHttpClient client) { + ConfigServerApiImpl(List<URI> configServerHosts, SelfCloseableHttpClient client) { this.configServerHosts = configServerHosts; this.client = client; } - public interface CreateRequest { + interface CreateRequest { HttpUriRequest createRequest(URI configServerUri) throws JsonProcessingException, UnsupportedEncodingException; } @@ -181,6 +153,10 @@ public class ConfigServerHttpRequestExecutor implements AutoCloseable { request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); } + public void setSSLConnectionSocketFactory(SSLConnectionSocketFactory sslSocketFactory) { + this.client = new SelfCloseableHttpClient(sslSocketFactory); + } + // Shuffle config server URIs to balance load private static List<URI> randomizeConfigServerUris(Collection<URI> configServerUris) { List<URI> shuffledConfigServerHosts = new ArrayList<>(configServerUris); @@ -188,62 +164,8 @@ public class ConfigServerHttpRequestExecutor implements AutoCloseable { return shuffledConfigServerHosts; } - private static SelfCloseableHttpClient createHttpClient(Optional<KeyStoreOptions> keyStoreOptions, - Optional<KeyStoreOptions> trustStoreOptions, - Optional<AthenzIdentity> athenzIdentity) { - NODE_ADMIN_LOGGER.info("Creating new HTTP client"); - try { - SSLContext sslContext = makeSslContext(keyStoreOptions, trustStoreOptions); - HostnameVerifier hostnameVerifier = makeHostnameVerifier(athenzIdentity); - SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier); - return new SelfCloseableHttpClient(sslSocketFactory); - } catch (Exception e) { - NODE_ADMIN_LOGGER.error("Failed to create HTTP client with custom SSL Context, proceeding with default", e); - return new SelfCloseableHttpClient(); - } - } - - private static SSLContext makeSslContext(Optional<KeyStoreOptions> keyStoreOptions, Optional<KeyStoreOptions> trustStoreOptions) { - AthenzSslContextBuilder sslContextBuilder = new AthenzSslContextBuilder(); - trustStoreOptions.ifPresent(options -> sslContextBuilder.withTrustStore(options.path.toFile(), options.type)); - keyStoreOptions.ifPresent(options -> { - try { - KeyStore keyStore = loadKeyStoreFromFileWithProvider(options.path, options.password, options.type, "BC"); - sslContextBuilder.withKeyStore(keyStore, options.password); - } catch (Exception e) { - throw new RuntimeException("Failed to read key store", e); - } - }); - - return sslContextBuilder.build(); - } - - private static HostnameVerifier makeHostnameVerifier(Optional<AthenzIdentity> athenzIdentity) { - return athenzIdentity - .map(identity -> (HostnameVerifier) new AthenzIdentityVerifier(Collections.singleton(identity))) - .orElse(SSLConnectionSocketFactory.getDefaultHostnameVerifier()); - } - @Override public void close() { - clientRefresherScheduler.shutdown(); - do { - try { - clientRefresherScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e1) { - NODE_ADMIN_LOGGER.info("Interrupted while waiting for clientRefresherScheduler to shutdown"); - } - } while (!clientRefresherScheduler.isTerminated()); - client.close(); } - - private static KeyStore loadKeyStoreFromFileWithProvider(Path path, char[] password, String keyStoreType, String provider) - throws IOException, GeneralSecurityException { - KeyStore keyStore = KeyStore.getInstance(keyStoreType, provider); - try (FileInputStream in = new FileInputStream(path.toFile())) { - keyStore.load(in, password); - } - return keyStore; - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java new file mode 100644 index 00000000000..f52487c306f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java @@ -0,0 +1,15 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; + +/** + * @author freva + */ +public interface ConfigServerClients { + NodeRepository nodeRepository(); + Orchestrator orchestrator(); + + void stop(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClientsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClientsImpl.java new file mode 100644 index 00000000000..43a2c66a9e5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClientsImpl.java @@ -0,0 +1,54 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorImpl; + +import java.util.Optional; + +/** + * @author freva + */ +public class ConfigServerClientsImpl implements ConfigServerClients { + + private final Optional<ConfigServerApi> configServerApi; + private final NodeRepository nodeRepository; + private final Orchestrator orchestrator; + + public ConfigServerClientsImpl(Environment environment) { + this(new SslConfigServerApiImpl(environment)); + } + + public ConfigServerClientsImpl(NodeRepository nodeRepository, Orchestrator orchestrator) { + this(nodeRepository, orchestrator, Optional.empty()); + } + + private ConfigServerClientsImpl(ConfigServerApi configServerApi) { + this(new NodeRepositoryImpl(configServerApi), new OrchestratorImpl(configServerApi), Optional.of(configServerApi)); + } + + private ConfigServerClientsImpl(NodeRepository nodeRepository, Orchestrator orchestrator, + Optional<ConfigServerApi> configServerApi) { + this.nodeRepository = nodeRepository; + this.orchestrator = orchestrator; + this.configServerApi = configServerApi; + } + + @Override + public NodeRepository nodeRepository() { + return nodeRepository; + } + + @Override + public Orchestrator orchestrator() { + return orchestrator; + } + + @Override + public void stop() { + configServerApi.ifPresent(ConfigServerApi::close); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java index 55d3ecc4e60..0a2ae1bd426 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/HttpException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +package com.yahoo.vespa.hosted.node.admin.configserver; import javax.ws.rs.core.Response; import java.util.Optional; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SelfCloseableHttpClient.java index 8e516729aff..cead7816387 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SelfCloseableHttpClient.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SelfCloseableHttpClient.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +package com.yahoo.vespa.hosted.node.admin.configserver; import com.yahoo.log.LogLevel; import org.apache.http.client.config.RequestConfig; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java new file mode 100644 index 00000000000..8c2b87f4068 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java @@ -0,0 +1,120 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.security.Security; +import java.util.Collections; +import java.util.Optional; + +/** + * ConfigServerApi with proper keystore, truststore and hostname verifier to communicate with the + * configserver(s). The keystore is refreshed automatically. + * + * @author freva + */ +public class SslConfigServerApiImpl implements ConfigServerApi { + + private final ConfigServerApiImpl configServerApi; + private final Environment environment; + private final Optional<ConfigServerKeyStoreRefresher> keyStoreRefresher; + + public SslConfigServerApiImpl(Environment environment) { + Security.addProvider(new BouncyCastleProvider()); + + this.environment = environment; + + // At this point we don't know the state of the keystore, it may not exist at all, or the keystore + // maybe exists, but the certificate in it is expired. Create the ConfigServerApi without a keystore + // (but with truststore and hostname verifier). + this.configServerApi = new ConfigServerApiImpl( + environment.getConfigServerUris(), makeSslConnectionSocketFactory(Optional.empty())); + + // If we have keystore options, we should make sure we use the keystore with the latest certificate, + // start the keystore refresher. + this.keyStoreRefresher = environment.getKeyStoreOptions().map(keyStoreOptions -> { + // Any callback from KeyStoreRefresher should result in using the latest keystore on disk + Runnable connectionFactoryRefresher = () -> configServerApi.setSSLConnectionSocketFactory( + makeSslConnectionSocketFactory(Optional.of(keyStoreOptions))); + + ConfigServerKeyStoreRefresher keyStoreRefresher = new ConfigServerKeyStoreRefresher( + keyStoreOptions, connectionFactoryRefresher, configServerApi); + + // Run the refresh once manually to make sure that we have a valid certificate, otherwise fail. + try { + keyStoreRefresher.refreshKeyStoreIfNeeded(); + connectionFactoryRefresher.run(); // Update connectionFactory with the keystore on disk + } catch (Exception e) { + throw new RuntimeException("Failed to acquire certificate to config server", e); + } + + keyStoreRefresher.start(); + return keyStoreRefresher; + }); + } + + @Override + public <T> T get(String path, Class<T> wantedReturnType) { + return configServerApi.get(path, wantedReturnType); + } + + @Override + public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { + return configServerApi.post(path, bodyJsonPojo, wantedReturnType); + } + + @Override + public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) { + return configServerApi.put(path, bodyJsonPojo, wantedReturnType); + } + + @Override + public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { + return configServerApi.patch(path, bodyJsonPojo, wantedReturnType); + } + + @Override + public <T> T delete(String path, Class<T> wantedReturnType) { + return configServerApi.delete(path, wantedReturnType); + } + + @Override + public void close() { + keyStoreRefresher.ifPresent(ConfigServerKeyStoreRefresher::stop); + configServerApi.close(); + } + + private SSLConnectionSocketFactory makeSslConnectionSocketFactory(Optional<KeyStoreOptions> keyStoreOptions) { + return new SSLConnectionSocketFactory(makeSslContext(keyStoreOptions), makeHostnameVerifier()); + } + + private SSLContext makeSslContext(Optional<KeyStoreOptions> keyStoreOptions) { + AthenzSslContextBuilder sslContextBuilder = new AthenzSslContextBuilder(); + environment.getTrustStoreOptions().ifPresent(options -> + sslContextBuilder.withTrustStore(options.path.toFile(), options.type)); + + keyStoreOptions.ifPresent(options -> { + try { + sslContextBuilder.withKeyStore(options.loadKeyStore(), options.password); + } catch (Exception e) { + throw new RuntimeException("Failed to read key store", e); + } + }); + + return sslContextBuilder.build(); + } + + private HostnameVerifier makeHostnameVerifier() { + return environment.getAthenzIdentity() + .map(identity -> (HostnameVerifier) new AthenzIdentityVerifier(Collections.singleton(identity))) + .orElseGet(SSLConnectionSocketFactory::getDefaultHostnameVerifier); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/CertificateSerializedPayload.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/CertificateSerializedPayload.java new file mode 100644 index 00000000000..e7148754fde --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/CertificateSerializedPayload.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.certificate; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.openssl.PEMParser; + +import java.io.IOException; +import java.io.StringReader; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +/** + * Contains PEM formatted signed certificate + * TODO: Combine with its counterpart in athenz-identity-provider-service? + * + * @author freva + */ +public class CertificateSerializedPayload { + + @JsonProperty("certificate") public final X509Certificate certificate; + + @JsonCreator + public CertificateSerializedPayload(@JsonProperty("certificate") @JsonDeserialize(using = CertificateDeserializer.class) + X509Certificate certificate) { + this.certificate = certificate; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CertificateSerializedPayload that = (CertificateSerializedPayload) o; + + return certificate.equals(that.certificate); + } + + @Override + public int hashCode() { + return certificate.hashCode(); + } + + @Override + public String toString() { + return "CertificateSerializedPayload{" + + "certificate='" + certificate + '\'' + + '}'; + } + + public static class CertificateDeserializer extends JsonDeserializer<X509Certificate> { + @Override + public X509Certificate deserialize( + JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + try (PEMParser pemParser = new PEMParser(new StringReader(jsonParser.getValueAsString()))) { + X509CertificateHolder x509CertificateHolder = (X509CertificateHolder) pemParser.readObject(); + return new JcaX509CertificateConverter().getCertificate(x509CertificateHolder); + } catch (CertificateException e) { + throw new RuntimeException("Failed to deserialize X509Certificate", e); + } + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java new file mode 100644 index 00000000000..bd71d661456 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresher.java @@ -0,0 +1,189 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.certificate; + +import com.yahoo.net.HostName; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Automatically refreshes the KeyStore used to authenticate this node to the configserver. + * The keystore contains a single certificate signed by one of the configservers. + * + * @author freva + */ +public class ConfigServerKeyStoreRefresher { + + private static final Logger logger = Logger.getLogger(ConfigServerKeyStoreRefresher.class.getName()); + private static final String KEY_STORE_ALIAS = "alias"; + static final long MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY = 3600; + static final String SIGNER_ALGORITHM = "SHA256withRSA"; + static final String CONFIG_SERVER_CERTIFICATE_SIGNING_PATH = "/athenz/v1/provider/sign"; + + private final ScheduledExecutorService executor; + private final KeyStoreOptions keyStoreOptions; + private final Runnable keyStoreUpdatedCallback; + private final ConfigServerApi configServerApi; + private final Clock clock; + private final String hostname; + + public ConfigServerKeyStoreRefresher( + KeyStoreOptions keyStoreOptions, Runnable keyStoreUpdatedCallback, ConfigServerApi configServerApi) { + this(keyStoreOptions, keyStoreUpdatedCallback, configServerApi, Executors.newScheduledThreadPool(0), + Clock.systemUTC(), HostName.getLocalhost()); + } + + ConfigServerKeyStoreRefresher(KeyStoreOptions keyStoreOptions, + Runnable keyStoreUpdatedCallback, + ConfigServerApi configServerApi, + ScheduledExecutorService executor, + Clock clock, + String hostname) { + this.keyStoreOptions = keyStoreOptions; + this.keyStoreUpdatedCallback = keyStoreUpdatedCallback; + this.configServerApi = configServerApi; + this.executor = executor; + this.clock = clock; + this.hostname = hostname; + } + + public void start() { + executor.schedule(this::refresh, getSecondsUntilNextRefresh(), TimeUnit.SECONDS); + } + + void refresh() { + try { + if (refreshKeyStoreIfNeeded()) { + keyStoreUpdatedCallback.run(); + } + final long secondsUntilNextRefresh = getSecondsUntilNextRefresh(); + executor.schedule(this::refresh, secondsUntilNextRefresh, TimeUnit.SECONDS); + logger.log(Level.INFO, "Successfully updated keystore, scheduled next refresh in " + + secondsUntilNextRefresh + "sec"); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to update keystore on schedule, will try again in " + + MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY + "sec", e); + executor.schedule(this::refresh, MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY, TimeUnit.SECONDS); + } + } + + public void stop() { + executor.shutdownNow(); + do { + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e1) { + logger.info("Interrupted while waiting for ConfigServerKeyStoreRefresher thread to shutdown"); + } + } while (!executor.isTerminated()); + } + + public boolean refreshKeyStoreIfNeeded() throws + IOException, NoSuchAlgorithmException, OperatorCreationException, CertificateException, KeyStoreException, NoSuchProviderException { + if (!shouldRefreshCertificate()) return false; + + KeyPair keyPair = generateKeyPair(); + PKCS10CertificationRequest csr = generateCsr(keyPair, hostname); + X509Certificate certificate = sendCsr(csr); + + storeCertificate(keyPair, certificate); + return true; + } + + private long getSecondsUntilNextRefresh() { + long secondsUntilNextCheck = 0; + try { + secondsUntilNextCheck = getSecondsUntilCertificateShouldBeRefreshed(); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to get remaining certificate lifetime", e); + } + + return Math.max(MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY, secondsUntilNextCheck); + } + + private boolean shouldRefreshCertificate() { + try { + return getSecondsUntilCertificateShouldBeRefreshed() <= 0; + } catch (Exception e) { // We can't read the key store for whatever reason, let's just try to refresh it + return true; + } + } + + /** + * Returns number of seconds until we should start trying to refresh the certificate, this should be + * well before the certificate actually expires so that we have enough time to retry without + * overloading config server. + */ + private long getSecondsUntilCertificateShouldBeRefreshed() + throws NoSuchAlgorithmException, CertificateException, NoSuchProviderException, KeyStoreException, IOException { + X509Certificate cert = getConfigServerCertificate(); + long notBefore = cert.getNotBefore().getTime() / 1000; + long notAfter = cert.getNotAfter().getTime() / 1000; + long now = clock.millis() / 1000; + long thirdOfLifetime = (notAfter - notBefore) / 3; + + return Math.max(0, notBefore + thirdOfLifetime - now); + } + + X509Certificate getConfigServerCertificate() throws NoSuchAlgorithmException, CertificateException, NoSuchProviderException, KeyStoreException, IOException { + return (X509Certificate) keyStoreOptions.loadKeyStore().getCertificate(KEY_STORE_ALIAS); + } + + private void storeCertificate(KeyPair keyPair, X509Certificate certificate) + throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException, NoSuchProviderException { + keyStoreOptions.path.getParent().toFile().mkdirs(); + X509Certificate[] certificateChain = {certificate}; + + try (FileOutputStream fos = new FileOutputStream(keyStoreOptions.path.toFile())) { + KeyStore keyStore = keyStoreOptions.getKeyStoreInstance(); + keyStore.load(null, null); + keyStore.setKeyEntry(KEY_STORE_ALIAS, keyPair.getPrivate(), keyStoreOptions.password, certificateChain); + keyStore.store(fos, keyStoreOptions.password); + } + } + + private X509Certificate sendCsr(PKCS10CertificationRequest csr) { + CertificateSerializedPayload certificateSerializedPayload = configServerApi.post( + CONFIG_SERVER_CERTIFICATE_SIGNING_PATH, + new CsrSerializedPayload(csr), + CertificateSerializedPayload.class); + + return certificateSerializedPayload.certificate; + } + + static KeyPair generateKeyPair() throws NoSuchAlgorithmException { + KeyPairGenerator rsa = KeyPairGenerator.getInstance("RSA"); + rsa.initialize(2048); + return rsa.genKeyPair(); + } + + private static PKCS10CertificationRequest generateCsr(KeyPair keyPair, String commonName) + throws NoSuchAlgorithmException, OperatorCreationException { + ContentSigner signer = new JcaContentSignerBuilder(SIGNER_ALGORITHM).build(keyPair.getPrivate()); + + return new JcaPKCS10CertificationRequestBuilder(new X500Name("CN=" + commonName), keyPair.getPublic()) + .build(signer); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/CsrSerializedPayload.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/CsrSerializedPayload.java new file mode 100644 index 00000000000..aa83fdc9e22 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/CsrSerializedPayload.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.certificate; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.bouncycastle.util.io.pem.PemObject; + +import java.io.IOException; +import java.io.StringWriter; + +/** + * Contains PEM formatted Certificate Signing Request (CSR) + * TODO: Combine with its counterpart in athenz-identity-provider-service? + * + * @author freva + */ +public class CsrSerializedPayload { + + @JsonProperty("csr") @JsonSerialize(using = CertificateRequestSerializer.class) + public final PKCS10CertificationRequest csr; + + @JsonCreator + public CsrSerializedPayload(@JsonProperty("csr") PKCS10CertificationRequest csr) { + this.csr = csr; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CsrSerializedPayload that = (CsrSerializedPayload) o; + + return csr.equals(that.csr); + } + + @Override + public int hashCode() { + return csr.hashCode(); + } + + @Override + public String toString() { + return "CsrSerializedPayload{" + + "csr='" + csr + '\'' + + '}'; + } + + public static class CertificateRequestSerializer extends JsonSerializer<PKCS10CertificationRequest> { + @Override + public void serialize( + PKCS10CertificationRequest csr, JsonGenerator gen, SerializerProvider serializers) throws IOException { + try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { + pemWriter.writeObject(new PemObject("CERTIFICATE REQUEST", csr.getEncoded())); + pemWriter.flush(); + gen.writeString(stringWriter.toString()); + } + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java index 9f4c6916b48..8012805f4d1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.vespa.hosted.node.admin.ContainerAclSpec; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; -import java.io.IOException; import java.util.List; import java.util.Optional; @@ -13,7 +12,7 @@ import java.util.Optional; * @author stiankri */ public interface NodeRepository { - List<ContainerNodeSpec> getContainersToRun(String baseHostName) throws IOException; + List<ContainerNodeSpec> getContainersToRun(String baseHostName); Optional<ContainerNodeSpec> getContainerNodeSpec(String hostName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryImpl.java index ab7332ca556..f2152dffc0c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryImpl.java @@ -1,22 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.vespa.hosted.node.admin.ContainerAclSpec; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.GetAclResponse; -import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.GetNodesResponse; -import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.NodeMessageResponse; -import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.UpdateNodeAttributesRequestBody; -import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.UpdateNodeAttributesResponse; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.HttpException; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetAclResponse; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetNodesResponse; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeMessageResponse; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.UpdateNodeAttributesRequestBody; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.UpdateNodeAttributesResponse; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.provision.Node; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -30,44 +29,37 @@ import java.util.stream.Collectors; public class NodeRepositoryImpl implements NodeRepository { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(NodeRepositoryImpl.class); - private final ConfigServerHttpRequestExecutor requestExecutor; + private final ConfigServerApi configServerApi; - public NodeRepositoryImpl(ConfigServerHttpRequestExecutor requestExecutor) { - this.requestExecutor = requestExecutor; + public NodeRepositoryImpl(ConfigServerApi configServerApi) { + this.configServerApi = configServerApi; } @Override - public List<ContainerNodeSpec> getContainersToRun(String baseHostName) throws IOException { - try { - final GetNodesResponse nodesForHost = requestExecutor.get( - "/nodes/v2/node/?parentHost=" + baseHostName + "&recursive=true", - GetNodesResponse.class); - - if (nodesForHost.nodes == null) { - throw new IOException("Response didn't contain nodes element"); - } - List<ContainerNodeSpec> nodes = new ArrayList<>(nodesForHost.nodes.size()); - for (GetNodesResponse.Node node : nodesForHost.nodes) { - ContainerNodeSpec nodeSpec; - try { - nodeSpec = createContainerNodeSpec(node); - } catch (IllegalArgumentException | NullPointerException e) { - NODE_ADMIN_LOGGER.warning("Bad node received from node repo when requesting children of the " - + baseHostName + " host: " + node, e); - continue; - } - nodes.add(nodeSpec); + public List<ContainerNodeSpec> getContainersToRun(String baseHostName) { + final GetNodesResponse nodesForHost = configServerApi.get( + "/nodes/v2/node/?parentHost=" + baseHostName + "&recursive=true", + GetNodesResponse.class); + + List<ContainerNodeSpec> nodes = new ArrayList<>(nodesForHost.nodes.size()); + for (GetNodesResponse.Node node : nodesForHost.nodes) { + ContainerNodeSpec nodeSpec; + try { + nodeSpec = createContainerNodeSpec(node); + } catch (IllegalArgumentException | NullPointerException e) { + NODE_ADMIN_LOGGER.warning("Bad node received from node repo when requesting children of the " + + baseHostName + " host: " + node, e); + continue; } - return nodes; - } catch (Exception e) { - throw new IOException(e); + nodes.add(nodeSpec); } + return nodes; } @Override public Optional<ContainerNodeSpec> getContainerNodeSpec(String hostName) { try { - GetNodesResponse.Node nodeResponse = requestExecutor.get("/nodes/v2/node/" + hostName, + GetNodesResponse.Node nodeResponse = configServerApi.get("/nodes/v2/node/" + hostName, GetNodesResponse.Node.class); if (nodeResponse == null) { return Optional.empty(); @@ -82,7 +74,7 @@ public class NodeRepositoryImpl implements NodeRepository { public List<ContainerAclSpec> getContainerAclSpecs(String hostName) { try { final String path = String.format("/nodes/v2/acl/%s?children=true", hostName); - final GetAclResponse response = requestExecutor.get(path, GetAclResponse.class); + final GetAclResponse response = configServerApi.get(path, GetAclResponse.class); return response.trustedNodes.stream() .map(node -> new ContainerAclSpec( node.hostname, node.ipAddress, ContainerName.fromHostname(node.trustedBy))) @@ -142,7 +134,7 @@ public class NodeRepositoryImpl implements NodeRepository { @Override public void updateNodeAttributes(final String hostName, final NodeAttributes nodeAttributes) { - UpdateNodeAttributesResponse response = requestExecutor.patch( + UpdateNodeAttributesResponse response = configServerApi.patch( "/nodes/v2/node/" + hostName, new UpdateNodeAttributesRequestBody(nodeAttributes), UpdateNodeAttributesResponse.class); @@ -166,7 +158,7 @@ public class NodeRepositoryImpl implements NodeRepository { } private void markNodeToState(String hostName, String state) { - NodeMessageResponse response = requestExecutor.put( + NodeMessageResponse response = configServerApi.put( "/nodes/v2/state/" + state + "/" + hostName, Optional.empty(), /* body */ NodeMessageResponse.class); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetAclResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java index 254ab5fa3ba..b7762cf6aa9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetAclResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java index ce31c3c1b4c..c94b3836100 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetNodesResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeMessageResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeMessageResponse.java index b8c903f863d..0f5e896c290 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeMessageResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeMessageResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/UpdateNodeAttributesRequestBody.java index 7acd94a2947..28605bc3a8d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/UpdateNodeAttributesRequestBody.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; import com.fasterxml.jackson.annotation.JsonInclude; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/UpdateNodeAttributesResponse.java index a11f3bf46a9..80c90e8311f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/UpdateNodeAttributesResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java new file mode 100644 index 00000000000..deb5352d1eb --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java index d98378e194c..b5d41b7fbb4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.orchestrator; +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; import java.util.List; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java index 51d542622da..fe19da0c41c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorException.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.orchestrator; +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; @SuppressWarnings("serial") public class OrchestratorException extends RuntimeException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java index 7093f3f12e7..0409004c6e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.orchestrator; +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import com.yahoo.vespa.hosted.node.admin.util.HttpException; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.orchestrator.restapi.HostApi; import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest; @@ -25,17 +25,17 @@ public class OrchestratorImpl implements Orchestrator { static final String ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API = ORCHESTRATOR_PATH_PREFIX + HostSuspensionApi.PATH_PREFIX; - private final ConfigServerHttpRequestExecutor requestExecutor; + private final ConfigServerApi configServerApi; - public OrchestratorImpl(ConfigServerHttpRequestExecutor requestExecutor) { - this.requestExecutor = requestExecutor; + public OrchestratorImpl(ConfigServerApi configServerApi) { + this.configServerApi = configServerApi; } @Override public void suspend(final String hostName) { UpdateHostResponse response; try { - response = requestExecutor.put(getSuspendPath(hostName), + response = configServerApi.put(getSuspendPath(hostName), Optional.empty(), /* body */ UpdateHostResponse.class); } catch (HttpException.NotFoundException n) { @@ -56,7 +56,7 @@ public class OrchestratorImpl implements Orchestrator { public void suspend(String parentHostName, List<String> hostNames) { final BatchOperationResult batchOperationResult; try { - batchOperationResult = requestExecutor.put( + batchOperationResult = configServerApi.put( ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class); @@ -77,7 +77,7 @@ public class OrchestratorImpl implements Orchestrator { UpdateHostResponse response; try { String path = getSuspendPath(hostName); - response = requestExecutor.delete(path, UpdateHostResponse.class); + response = configServerApi.delete(path, UpdateHostResponse.class); } catch (HttpException.NotFoundException n) { throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found"); } catch (HttpException e) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorNotFoundException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java index 088152bd7a3..ac39f7c3280 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorNotFoundException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorNotFoundException.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.orchestrator; +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; @SuppressWarnings("serial") public class OrchestratorNotFoundException extends OrchestratorException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java new file mode 100644 index 00000000000..6f67fe2ea08 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java new file mode 100644 index 00000000000..347f6eaf516 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 88f5c9acfed..b30cac2476e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -12,8 +12,8 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImpl; import com.yahoo.vespa.hosted.dockerapi.DockerNetworkCreator; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.NATCommand; -import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.io.IOException; @@ -45,6 +45,9 @@ public class DockerOperationsImpl implements DockerOperations { private static final String MANAGER_NAME = "node-admin"; + private static final String LOCAL_IPV6_PREFIX = "fd00::"; + private static final String DOCKER_CUSTOM_BRIDGE_NETWORK_NAME = "vespa-bridge"; + // Map of directories to mount and whether they should be writable by everyone private static final Map<String, Boolean> DIRECTORIES_TO_MOUNT = new HashMap<>(); @@ -63,7 +66,7 @@ public class DockerOperationsImpl implements DockerOperations { DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/yms_agent"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/ysar"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/ystatus"), false); - DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/zpe_policy_updater"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/zpu"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/cache"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/crash"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/db/jdisc"), false); @@ -80,6 +83,7 @@ public class DockerOperationsImpl implements DockerOperations { DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/yca"), true); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/ycore++"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/zookeeper"), false); + DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("var/zpe"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("tmp"), false); } @@ -119,11 +123,16 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc. .withAddCapability("SYS_ADMIN"); // Needed for perf - if (!docker.networkNATed()) { - logger.info("Network not nated - setting up with specific ip address on a macvlan"); + if (!docker.networkNPTed()) { command.withIpAddress(nodeInetAddress); command.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME); command.withVolume("/etc/hosts", "/etc/hosts"); // TODO This is probably not nessesary - review later + } else { + command.withIpAddress(NetworkPrefixTranslator.translate( + nodeInetAddress, + InetAddress.getByName(LOCAL_IPV6_PREFIX), + 64)); + command.withNetworkMode(DOCKER_CUSTOM_BRIDGE_NETWORK_NAME); } for (String pathInNode : DIRECTORIES_TO_MOUNT.keySet()) { @@ -143,17 +152,14 @@ public class DockerOperationsImpl implements DockerOperations { command.create(); if (isIPv6) { - if (!docker.networkNATed()) { + if (!docker.networkNPTed()) { docker.connectContainerToNetwork(containerName, "bridge"); } docker.startContainer(containerName); - setupContainerNetworkConnectivity(containerName, nodeInetAddress); + setupContainerNetworkConnectivity(containerName); } else { docker.startContainer(containerName); - if (docker.networkNATed()) { - setupContainerNetworkConnectivity(containerName, nodeInetAddress); - } } DIRECTORIES_TO_MOUNT.entrySet().stream().filter(Map.Entry::getValue).forEach(entry -> @@ -175,7 +181,7 @@ public class DockerOperationsImpl implements DockerOperations { logger.info("Deleting container " + containerName.asString()); docker.deleteContainer(containerName); - if (docker.networkNATed()) { + if (docker.networkNPTed()) { logger.info("Delete iptables NAT rules for " + containerName.asString()); try { InetAddress nodeInetAddress = environment.getInetAddressForHost(nodeSpec.hostname); @@ -221,16 +227,12 @@ public class DockerOperationsImpl implements DockerOperations { /** * For macvlan: + * <p> * Due to a bug in docker (https://github.com/docker/libnetwork/issues/1443), we need to manually set * IPv6 gateway in containers connected to more than one docker network - * - * For nat: - * Setup iptables NAT rules to map the hosts public ips to the containers */ - private void setupContainerNetworkConnectivity(ContainerName containerName, InetAddress externalAddress) throws IOException { - if (docker.networkNATed()) { - insertNAT(containerName, externalAddress); - } else { + private void setupContainerNetworkConnectivity(ContainerName containerName) throws IOException { + if (!docker.networkNPTed()) { InetAddress hostDefaultGateway = DockerNetworkCreator.getDefaultGatewayLinux(true); executeCommandInNetworkNamespace(containerName, "route", "-A", "inet6", "add", "default", "gw", hostDefaultGateway.getHostAddress(), "dev", "eth1"); @@ -274,7 +276,7 @@ public class DockerOperationsImpl implements DockerOperations { final String[] wrappedCommand = Stream.concat( Stream.of("sudo", "nsenter", String.format("--net=/host/proc/%d/ns/net", containerPid), "--"), Stream.of(command)) - .toArray(String[]::new); + .toArray(String[]::new); try { Pair<Integer, String> result = processExecuter.exec(wrappedCommand); @@ -326,25 +328,4 @@ public class DockerOperationsImpl implements DockerOperations { public void deleteUnusedDockerImages() { docker.deleteUnusedDockerImages(); } - - /** - * Only insert NAT rules if they don't exist (or else they compounded) - */ - private void insertNAT(ContainerName containerName, InetAddress externalAddress) throws IOException { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - String ipv6Str = docker.getGlobalIPv6Address(containerName); - - // Check if exist - String checkCommand = NATCommand.check(externalAddress, InetAddress.getByName(ipv6Str)); - Pair<Integer, String> result = processExecuter.exec(checkCommand); - if (result.getFirst() == 0 ) return; - - // Setup NAT - String natCommand = NATCommand.insert(externalAddress, InetAddress.getByName(ipv6Str)); - logger.info("Setting up NAT rules: " + natCommand); - result = processExecuter.exec(checkCommand); - if (result.getFirst() != 0 ) { - throw new IOException("Unable to setup NAT rule - error message: " + result.getSecond()); - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java new file mode 100644 index 00000000000..a52dedb90e5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslator.java @@ -0,0 +1,38 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author smorgrav + */ +package com.yahoo.vespa.hosted.node.admin.docker; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +class NetworkPrefixTranslator { + + /** + * For NPTed networks we want to find the private address from a public. + * + * @param address The original address to translate + * @param prefix The prefix address + * @param subnetSize in bits - e.g a /64 subnet equals 64 bits + * @return The translated address + */ + static Inet6Address translate(InetAddress address, InetAddress prefix, int subnetSize) { + + byte[] originalAddress = address.getAddress(); + byte[] prefixAddress = prefix.getAddress(); + byte[] translatedAddress = new byte[16]; + + for (int i = 0; i < 16; i++) { + translatedAddress[i] = i < subnetSize / 8 ? prefixAddress[i] : originalAddress[i]; + } + + try { + return (Inet6Address) InetAddress.getByAddress(address.getHostName(), translatedAddress); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java index dde2a39da67..422e47fe83a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.logging; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import java.util.Optional; import java.util.stream.Collectors; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 257247bf44a..c12f1168240 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker; @@ -223,6 +223,11 @@ public class StorageMaintainer { } private void addHandleCoredumpsCommand(MaintainerExecutor maintainerExecutor, ContainerName containerName, ContainerNodeSpec nodeSpec) { + if (!environment.getCoredumpFeedEndpoint().isPresent()) { + // Core dump handling is disabled. + return; + } + Map<String, Object> attributes = new HashMap<>(); attributes.put("hostname", nodeSpec.hostname); attributes.put("parent_hostname", HostName.getLocalhost()); @@ -243,7 +248,7 @@ public class StorageMaintainer { .withArgument("doneCoredumpsPath", environment.pathInNodeAdminToDoneCoredumps()) .withArgument("coredumpsPath", environment.pathInNodeAdminFromPathInNode( containerName, getDefaults().underVespaHome("var/crash"))) - .withArgument("feedEndpoint", environment.getCoredumpFeedEndpoint()) + .withArgument("feedEndpoint", environment.getCoredumpFeedEndpoint().get()) .withArgument("attributes", attributes); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 2947ef68ba4..a453ea46ffd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.Chain; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.Command; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.FlushCommand; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.iptables.PolicyCommand; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.util.HashMap; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java index b0631fc712b..d19f64a2bc3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java @@ -8,7 +8,10 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.component.AdminComponent; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.node.admin.component.DockerAdminComponent; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerClientsImpl; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import java.io.File; @@ -59,7 +62,11 @@ public class NodeAdminMain implements AutoCloseable { private AdminComponent selectAdminComponent(NodeAdminConfig config) { if (config.mainComponent == null) { - return new DockerAdminComponent(configServerConfig, config, docker, metricReceiver, classLocking); + return new DockerAdminComponent(configServerConfig, + docker, + metricReceiver, + classLocking, + new ConfigServerClientsImpl(new Environment(configServerConfig))); } logger.log(LogLevel.INFO, () -> { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java index e4e66b57186..8f6e5bf9748 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java @@ -7,16 +7,15 @@ import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.concurrent.classlock.LockInterruptException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.util.HttpException; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.provision.Node; -import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -64,8 +63,8 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private final String dockerHostHostName; private final Duration nodeAdminConvergeStateInterval; - private final ClassLocking classLocking; - private Optional<ClassLock> classLock; + private final Optional<ClassLocking> classLocking; + private Optional<ClassLock> classLock = Optional.empty(); private Instant lastTick; public NodeAdminStateUpdaterImpl( @@ -76,7 +75,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { String dockerHostHostName, Clock clock, Duration nodeAdminConvergeStateInterval, - ClassLocking classLocking) { + Optional<ClassLocking> classLocking) { log.info(objectToString() + ": Creating object"); this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; @@ -88,12 +87,14 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { this.lastTick = clock.instant(); this.loopThread = new Thread(() -> { - log.info(objectToString() + ": Acquiring lock"); - try { - classLock = Optional.of(classLocking.lockWhile(NodeAdminStateUpdater.class, () -> !terminated.get())); - } catch (LockInterruptException e) { - classLock = Optional.empty(); - return; + if (classLocking.isPresent()) { + log.info(objectToString() + ": Acquiring lock"); + try { + classLock = Optional.of(classLocking.get().lockWhile(NodeAdminStateUpdater.class, () -> !terminated.get())); + } catch (LockInterruptException e) { + classLock = Optional.empty(); + return; + } } log.info(objectToString() + ": Starting threads and schedulers"); @@ -267,35 +268,22 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { log.info("Frozen, skipping fetching info from node repository"); return; } - final List<ContainerNodeSpec> containersToRun; - try { - containersToRun = nodeRepository.getContainersToRun(dockerHostHostName); - } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed fetching container info from node repository", e); - return; - } - if (containersToRun == null) { - log.warning("Got null from node repository"); - return; - } + try { + final List<ContainerNodeSpec> containersToRun = nodeRepository.getContainersToRun(dockerHostHostName); nodeAdmin.refreshContainersToRun(containersToRun); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed updating node admin: ", e); + log.log(LogLevel.WARNING, "Failed to update which containers should be running", e); } } } private List<String> getNodesInActiveState() { - try { - return nodeRepository.getContainersToRun(dockerHostHostName) - .stream() - .filter(nodespec -> nodespec.nodeState == Node.State.active) - .map(nodespec -> nodespec.hostname) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new RuntimeException("Failed to get nodes from node repo", e); - } + return nodeRepository.getContainersToRun(dockerHostHostName) + .stream() + .filter(nodespec -> nodespec.nodeState == Node.State.active) + .map(nodespec -> nodespec.hostname) + .collect(Collectors.toList()); } public void start() { @@ -308,7 +296,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { throw new RuntimeException("Can not re-stop a node agent."); } - classLocking.interrupt(); + classLocking.ifPresent(ClassLocking::interrupt); // First we need to stop NodeAdminStateUpdaterImpl thread to make sure no new NodeAgents are spawned signalWorkToBeDone(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index edf4f059fc2..893054e5ac0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -17,10 +17,10 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.provision.Node; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/package-info.java new file mode 100644 index 00000000000..948dfc070bc --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java index 9a8ed19e0ac..9eaa2395d21 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java @@ -25,7 +25,7 @@ import java.util.stream.Collectors; */ public class CommandLine { private static Logger logger = Logger.getLogger(CommandLine.class.getName()); - private static Pattern UNESCAPED_ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$"); + private static Pattern UNESCAPED_ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=!@%/+:.,_-]+$"); /** The default timeout. See setTimeout() for details. */ public static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(10); @@ -74,7 +74,7 @@ public class CommandLine { /** Add arguments by splitting arguments by space. */ public CommandLine addTokens(String arguments) { - return add(arguments.split(" ")); + return add(arguments.split("\\s+")); } /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java index 643abde101b..1115f6dca91 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/KeyStoreOptions.java @@ -1,16 +1,45 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.util; +import java.io.FileInputStream; +import java.io.IOException; import java.nio.file.Path; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.cert.CertificateException; +import java.util.Optional; public class KeyStoreOptions { public final Path path; public final char[] password; public final String type; + private final Optional<String> provider; public KeyStoreOptions(Path path, char[] password, String type) { + this(path, password, type, null); + } + + public KeyStoreOptions(Path path, char[] password, String type, String provider) { this.path = path; this.password = password; this.type = type; + this.provider = Optional.ofNullable(provider); + } + + public KeyStore loadKeyStore() + throws IOException, NoSuchProviderException, KeyStoreException, CertificateException, NoSuchAlgorithmException { + try (FileInputStream in = new FileInputStream(path.toFile())) { + KeyStore keyStore = getKeyStoreInstance(); + keyStore.load(in, password); + return keyStore; + } + } + + public KeyStore getKeyStoreInstance() throws NoSuchProviderException, KeyStoreException { + return provider.isPresent() ? + KeyStore.getInstance(type, provider.get()) : + KeyStore.getInstance(type); } } diff --git a/node-admin/src/main/sh/node-admin.sh b/node-admin/src/main/sh/node-admin.sh index 3196ff9fa32..b2dac920c56 100755 --- a/node-admin/src/main/sh/node-admin.sh +++ b/node-admin/src/main/sh/node-admin.sh @@ -67,6 +67,14 @@ EOF exit 1 } +Start() { + "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root "$@" +} + +Stop() { + "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root "$@" +} + if (( $# == 0 )); then Usage fi @@ -75,11 +83,11 @@ command="$1" shift case "$command" in - start) - "$VESPA_HOME"/libexec/vespa/standalone-container.sh start -s node-admin -u root "$@" - ;; - stop) - "$VESPA_HOME"/libexec/vespa/standalone-container.sh stop -s node-admin -u root "$@" + start) Start "$@" ;; + stop) Stop "$@" ;; + restart) + Stop "$@" + Start "$@" ;; *) Usage ;; esac diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index 175d3a9a051..f39a64d2dee 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.util; +package com.yahoo.vespa.hosted.node.admin.configserver; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -34,7 +34,7 @@ import static org.mockito.Mockito.when; * * @author dybis */ -public class ConfigServerHttpRequestExecutorTest { +public class ConfigServerApiImplTest { @JsonIgnoreProperties(ignoreUnknown = true) public static class TestPojo { @@ -49,7 +49,7 @@ public class ConfigServerHttpRequestExecutorTest { private final List<URI> configServers = Arrays.asList(URI.create(uri1), URI.create(uri2)); private final StringBuilder mockLog = new StringBuilder(); - private ConfigServerHttpRequestExecutor executor; + private ConfigServerApiImpl executor; private int mockReturnCode = 200; @Before @@ -72,7 +72,7 @@ public class ConfigServerHttpRequestExecutorTest { return response; }); - executor = new ConfigServerHttpRequestExecutor(configServers, httpMock); + executor = new ConfigServerApiImpl(configServers, httpMock); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java new file mode 100644 index 00000000000..f9f8b230154 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherTest.java @@ -0,0 +1,162 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.certificate; + +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.Date; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +/** + * @author freva + */ +public class ConfigServerKeyStoreRefresherTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private final ManualClock clock = new ManualClock(); + private final String commonName = "CertificateRefresherTest"; + private final Duration certificateExpiration = Duration.ofDays(6); + private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); + private final Runnable keyStoreUpdatedCallback = mock(Runnable.class); + private final ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + private KeyStoreOptions keyStoreOptions; + + @Before + public void setup() { + keyStoreOptions = new KeyStoreOptions( + tempFolder.getRoot().toPath().resolve("some/path/keystore.p12"), new char[0], "PKCS12", null); + } + + @Test + public void manually_trigger_certificate_refresh() throws Exception { + X509Certificate firstCertificate = mockConfigServerCertificateSigning(1); + + ConfigServerKeyStoreRefresher keyStoreRefresher = new ConfigServerKeyStoreRefresher( + keyStoreOptions, keyStoreUpdatedCallback, configServerApi, executor, clock, commonName); + + // No keystore previously existed, so a new one should be written + assertTrue(keyStoreRefresher.refreshKeyStoreIfNeeded()); + assertEquals(firstCertificate, keyStoreRefresher.getConfigServerCertificate()); + + // Calling it again before a third of certificate lifetime has passed has no effect + assertFalse(keyStoreRefresher.refreshKeyStoreIfNeeded()); + assertEquals(firstCertificate, keyStoreRefresher.getConfigServerCertificate()); + + // After a third of the expiration time passes, we should refresh the certificate + clock.advance(certificateExpiration.dividedBy(3).plusSeconds(1)); + X509Certificate secondCertificate = mockConfigServerCertificateSigning(2); + assertTrue(keyStoreRefresher.refreshKeyStoreIfNeeded()); + assertEquals(secondCertificate, keyStoreRefresher.getConfigServerCertificate()); + + verify(configServerApi, times(2)) + .post(eq(ConfigServerKeyStoreRefresher.CONFIG_SERVER_CERTIFICATE_SIGNING_PATH), any(), any()); + + // We're just triggering refresh manually, so callback and executor should not have been touched + verifyZeroInteractions(keyStoreUpdatedCallback); + verifyZeroInteractions(executor); + } + + @Test + public void certificate_refresh_schedule_test() throws Exception { + ConfigServerKeyStoreRefresher keyStoreRefresher = new ConfigServerKeyStoreRefresher( + keyStoreOptions, keyStoreUpdatedCallback, configServerApi, executor, clock, commonName); + + // No keystore exist, so refresh once + mockConfigServerCertificateSigning(1); + assertTrue(keyStoreRefresher.refreshKeyStoreIfNeeded()); + + // Start automatic refreshment, since keystore was just written, next check should be in 1/3rd of + // certificate lifetime, which is in 2 days. + keyStoreRefresher.start(); + Duration nextExpectedExecution = Duration.ofDays(2); + verify(executor, times(1)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS)); + + // First automatic refreshment goes without any problems + clock.advance(nextExpectedExecution); + mockConfigServerCertificateSigning(2); + keyStoreRefresher.refresh(); + verify(executor, times(2)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS)); + verify(keyStoreUpdatedCallback).run(); + + // We fail to refresh the certificate, wait minimum amount of time and try again + clock.advance(nextExpectedExecution); + mockConfigServerCertificateSigningFailure(new RuntimeException()); + keyStoreRefresher.refresh(); + nextExpectedExecution = Duration.ofSeconds(ConfigServerKeyStoreRefresher.MINIMUM_SECONDS_BETWEEN_REFRESH_RETRY); + verify(executor, times(1)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS)); + + clock.advance(nextExpectedExecution); + keyStoreRefresher.refresh(); + verify(executor, times(2)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS)); + verifyNoMoreInteractions(keyStoreUpdatedCallback); // Callback not called after the last 2 failures + + clock.advance(nextExpectedExecution); + mockConfigServerCertificateSigning(3); + keyStoreRefresher.refresh(); + nextExpectedExecution = Duration.ofDays(2); + verify(executor, times(3)).schedule(any(Runnable.class), eq(nextExpectedExecution.getSeconds()), eq(TimeUnit.SECONDS)); + verify(keyStoreUpdatedCallback, times(2)).run(); + } + + private X509Certificate mockConfigServerCertificateSigning(int serial) throws Exception { + X509Certificate certificate = makeCertificate(serial); + + when(configServerApi.post(eq(ConfigServerKeyStoreRefresher.CONFIG_SERVER_CERTIFICATE_SIGNING_PATH), any(), any())) + .thenReturn(new CertificateSerializedPayload(certificate)); + return certificate; + } + + private void mockConfigServerCertificateSigningFailure(Exception exception) throws Exception { + when(configServerApi.post(eq(ConfigServerKeyStoreRefresher.CONFIG_SERVER_CERTIFICATE_SIGNING_PATH), any(), any())) + .thenThrow(exception); + } + + private X509Certificate makeCertificate(int serial) throws Exception { + try { + KeyPair keyPair = ConfigServerKeyStoreRefresher.generateKeyPair(); + X500Name subject = new X500Name("CN=" + commonName); + Date notBefore = Date.from(clock.instant()); + Date notAfter = Date.from(clock.instant().plus(certificateExpiration)); + + JcaX509v3CertificateBuilder certGen = new JcaX509v3CertificateBuilder(subject, + BigInteger.valueOf(serial), notBefore, notAfter, subject, keyPair.getPublic()); + ContentSigner sigGen = new JcaContentSignerBuilder(ConfigServerKeyStoreRefresher.SIGNER_ALGORITHM) + .build(keyPair.getPrivate()); + return new JcaX509CertificateConverter() + .setProvider(new BouncyCastleProvider()) + .getCertificate(certGen.build(sigGen)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryImplTest.java index 949b4ccdf78..85e101714e8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepositoryImplTest.java @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.noderepository; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; @@ -36,7 +36,7 @@ import static org.junit.Assert.fail; */ public class NodeRepositoryImplTest { private JDisc container; - private ConfigServerHttpRequestExecutor requestExecutor; + private ConfigServerApiImpl configServerApi; private int findRandomOpenPort() throws IOException { @@ -63,8 +63,7 @@ public class NodeRepositoryImplTest { try { final int port = findRandomOpenPort(); container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable); - requestExecutor = ConfigServerHttpRequestExecutor.create( - Collections.singleton(URI.create("http://127.0.0.1:" + port)), Optional.empty(), Optional.empty(), Optional.empty()); + configServerApi = new ConfigServerApiImpl(Collections.singleton(URI.create("http://127.0.0.1:" + port))); return; } catch (RuntimeException e) { lastException = e; @@ -75,7 +74,7 @@ public class NodeRepositoryImplTest { private void waitForJdiscContainerToServe() throws InterruptedException { Instant start = Instant.now(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); while (Instant.now().minusSeconds(120).isBefore(start)) { try { nodeRepositoryApi.getContainersToRun("foobar"); @@ -95,9 +94,9 @@ public class NodeRepositoryImplTest { } @Test - public void testGetContainersToRunApi() throws IOException, InterruptedException { + public void testGetContainersToRunApi() throws InterruptedException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); String dockerHostHostname = "dockerhost1.yahoo.com"; final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(dockerHostHostname); @@ -116,7 +115,7 @@ public class NodeRepositoryImplTest { @Test public void testGetContainer() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); String hostname = "host4.yahoo.com"; Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname); assertThat(nodeSpec.isPresent(), is(true)); @@ -126,7 +125,7 @@ public class NodeRepositoryImplTest { @Test public void testGetContainerForNonExistingNode() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); String hostname = "host-that-does-not-exist"; Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname); assertFalse(nodeSpec.isPresent()); @@ -135,7 +134,7 @@ public class NodeRepositoryImplTest { @Test public void testUpdateNodeAttributes() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); String hostname = "host4.yahoo.com"; nodeRepositoryApi.updateNodeAttributes( hostname, @@ -148,7 +147,7 @@ public class NodeRepositoryImplTest { @Test(expected = RuntimeException.class) public void testUpdateNodeAttributesWithBadValue() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); String hostname = "host4.yahoo.com"; nodeRepositoryApi.updateNodeAttributes( hostname, @@ -160,7 +159,7 @@ public class NodeRepositoryImplTest { @Test public void testMarkAsReady() throws InterruptedException, IOException { - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerApi); waitForJdiscContainerToServe(); nodeRepositoryApi.markAsDirty("host5.yahoo.com"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java index 779a0a6a376..2d355c93c09 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.orchestrator; +package com.yahoo.vespa.hosted.node.admin.configserver.orchestrator; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.HttpException; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; +import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest; import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; @@ -23,12 +23,12 @@ import static org.mockito.Mockito.when; public class OrchestratorImplTest { private static final String hostName = "host123.yahoo.com"; - private final ConfigServerHttpRequestExecutor requestExecutor = mock(ConfigServerHttpRequestExecutor.class); - private final OrchestratorImpl orchestrator = new OrchestratorImpl(requestExecutor); + private final ConfigServerApiImpl configServerApi = mock(ConfigServerApiImpl.class); + private final OrchestratorImpl orchestrator = new OrchestratorImpl(configServerApi); @Test public void testSuspendCall() { - when(requestExecutor.put( + when(configServerApi.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", Optional.empty(), UpdateHostResponse.class @@ -39,7 +39,7 @@ public class OrchestratorImplTest { @Test(expected=OrchestratorException.class) public void testSuspendCallWithFailureReason() { - when(requestExecutor.put( + when(configServerApi.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", Optional.empty(), UpdateHostResponse.class @@ -50,7 +50,7 @@ public class OrchestratorImplTest { @Test(expected=OrchestratorNotFoundException.class) public void testSuspendCallWithNotFound() { - when(requestExecutor.put( + when(configServerApi.put( any(String.class), any(), any() @@ -61,7 +61,7 @@ public class OrchestratorImplTest { @Test(expected=RuntimeException.class) public void testSuspendCallWithSomeOtherException() { - when(requestExecutor.put( + when(configServerApi.put( any(String.class), any(), any() @@ -73,7 +73,7 @@ public class OrchestratorImplTest { @Test public void testResumeCall() { - when(requestExecutor.delete( + when(configServerApi.delete( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, null)); @@ -83,7 +83,7 @@ public class OrchestratorImplTest { @Test(expected=OrchestratorException.class) public void testResumeCallWithFailureReason() { - when(requestExecutor.delete( + when(configServerApi.delete( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); @@ -93,7 +93,7 @@ public class OrchestratorImplTest { @Test(expected=OrchestratorNotFoundException.class) public void testResumeCallWithNotFound() { - when(requestExecutor.delete( + when(configServerApi.delete( any(String.class), any() )).thenThrow(new HttpException.NotFoundException("Not Found")); @@ -103,7 +103,7 @@ public class OrchestratorImplTest { @Test(expected=RuntimeException.class) public void testResumeCallWithSomeOtherException() { - when(requestExecutor.put( + when(configServerApi.put( any(String.class), any(), any() @@ -118,7 +118,7 @@ public class OrchestratorImplTest { String parentHostName = "host1.test.yahoo.com"; List<String> hostNames = Arrays.asList("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); - when(requestExecutor.put( + when(configServerApi.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class @@ -133,7 +133,7 @@ public class OrchestratorImplTest { List<String> hostNames = Arrays.asList("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); String failureReason = "Failed to suspend"; - when(requestExecutor.put( + when(configServerApi.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class @@ -148,7 +148,7 @@ public class OrchestratorImplTest { List<String> hostNames = Arrays.asList("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); String exceptionMessage = "Exception: Something crashed!"; - when(requestExecutor.put( + when(configServerApi.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 616a18d2f2f..034c9352a10 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import org.junit.Test; import org.mockito.InOrder; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java new file mode 100644 index 00000000000..96afe685a61 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/NetworkPrefixTranslatorTest.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author smorgrav + */ +package com.yahoo.vespa.hosted.node.admin.docker; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +public class NetworkPrefixTranslatorTest { + + @Test + public void translator_with_valid_parameters() throws UnknownHostException { + + // Test simplest possible address + Inet6Address original = (Inet6Address)InetAddress.getByName("2001:db8::1"); + Inet6Address prefix = (Inet6Address)InetAddress.getByName("fd00::"); + Inet6Address translated = NetworkPrefixTranslator.translate(original, prefix, 64); + Assert.assertEquals("fd00:0:0:0:0:0:0:1", translated.getHostAddress()); + + + // Test an actual aws address we use + original = (Inet6Address)InetAddress.getByName("2600:1f16:f34:5300:ccc6:1703:b7c2:369d"); + translated = NetworkPrefixTranslator.translate(original, prefix, 64); + Assert.assertEquals("fd00:0:0:0:ccc6:1703:b7c2:369d", translated.getHostAddress()); + + // Test different subnet size + translated = NetworkPrefixTranslator.translate(original, prefix, 48); + Assert.assertEquals("fd00:0:0:5300:ccc6:1703:b7c2:369d", translated.getHostAddress()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 3cfc67824ed..d22cb968969 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -14,15 +14,16 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; +import java.util.Optional; import java.util.function.Function; import static org.mockito.Matchers.any; @@ -67,7 +68,8 @@ public class DockerTester implements AutoCloseable { orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, storageMaintainer, - nodeAdmin, "basehostname", clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, new ClassLocking()); + nodeAdmin, "basehostname", clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, + Optional.of(new ClassLocking())); nodeAdminStateUpdater.start(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index 2d2a622f8f0..8557da75ee9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -4,10 +4,9 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.vespa.hosted.node.admin.ContainerAclSpec; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.provision.Node; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -32,7 +31,7 @@ public class NodeRepoMock implements NodeRepository { } @Override - public List<ContainerNodeSpec> getContainersToRun(String dockerHostHostname) throws IOException { + public List<ContainerNodeSpec> getContainersToRun(String dockerHostHostname) { synchronized (monitor) { return new ArrayList<>(containerNodeSpecsByHostname.values()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java index dc285bb27ce..469022cec56 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import java.util.List; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index a0e122d99fc..3a163c94caa 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -15,14 +15,14 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; +import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.provision.Node; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; @@ -46,6 +46,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collections; +import java.util.Optional; import java.util.function.Function; import java.util.logging.Logger; @@ -243,7 +244,8 @@ public class RunInContainerTest { storageMaintainer, aclMaintainer, environment, Clock.systemUTC(), NODE_AGENT_SCAN_INTERVAL); private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); private final NodeAdminStateUpdaterImpl nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, - orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", Clock.systemUTC(), NODE_ADMIN_CONVERGE_STATE_INTERVAL, new ClassLocking()); + orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", + Clock.systemUTC(), NODE_ADMIN_CONVERGE_STATE_INTERVAL, Optional.of(new ClassLocking())); public NodeAdminProviderWithMocks() { nodeAdminStateUpdater.start(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index 67627ee1a83..443948db450 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import java.time.Clock; import java.util.Optional; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java index 399579a7ce7..ad50041ab69 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.logging; import com.google.common.collect.ImmutableList; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 38dd11a7a51..6aeafebaea7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Rule; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index a699377b4c3..d50f869617a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.ContainerAclSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import org.junit.Before; import org.junit.Test; import org.mockito.verification.VerificationMode; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java index 7920d0cad29..c9985247018 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java @@ -4,17 +4,17 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; -import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -47,11 +47,12 @@ public class NodeAdminStateUpdaterImplTest { private final Duration convergeStateInterval = Duration.ofSeconds(30); private final NodeAdminStateUpdaterImpl refresher = spy(new NodeAdminStateUpdaterImpl( - nodeRepository, orchestrator, storageMaintainer, nodeAdmin, parentHostname, clock, convergeStateInterval, null)); + nodeRepository, orchestrator, storageMaintainer, nodeAdmin, parentHostname, clock, + convergeStateInterval, Optional.empty())); @Test - public void testStateConvergence() throws IOException { + public void testStateConvergence() { mockNodeRepo(4); List<String> activeHostnames = nodeRepository.getContainersToRun(parentHostname).stream() .map(node -> node.hostname) @@ -153,7 +154,7 @@ public class NodeAdminStateUpdaterImplTest { } @Test - public void half_transition_revert() throws IOException { + public void half_transition_revert() { mockNodeRepo(3); // Initially everything is frozen to force convergence @@ -180,7 +181,7 @@ public class NodeAdminStateUpdaterImplTest { verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Make sure that we unfreeze! } - private void mockNodeRepo(int numberOfNodes) throws IOException { + private void mockNodeRepo(int numberOfNodes) { List<ContainerNodeSpec> containersToRun = IntStream.range(0, numberOfNodes) .mapToObj(i -> new ContainerNodeSpec.Builder() .hostname("host" + i + ".test.yahoo.com") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 8067bf9ba69..fb9303ea382 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -15,11 +15,11 @@ import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import org.mockito.InOrder; @@ -95,7 +95,7 @@ public class NodeAgentImplTest { @Test - public void upToDateContainerIsUntouched() throws Exception { + public void upToDateContainerIsUntouched() { final long restartGeneration = 1; final long rebootGeneration = 0; final ContainerNodeSpec nodeSpec = nodeSpecBuilder @@ -135,7 +135,7 @@ public class NodeAgentImplTest { } @Test - public void verifyRemoveOldFilesIfDiskFull() throws Exception { + public void verifyRemoveOldFilesIfDiskFull() { final long restartGeneration = 1; final long rebootGeneration = 0; final ContainerNodeSpec nodeSpec = nodeSpecBuilder @@ -199,7 +199,7 @@ public class NodeAgentImplTest { } @Test - public void containerIsNotStoppedIfNewImageMustBePulled() throws Exception { + public void containerIsNotStoppedIfNewImageMustBePulled() { final DockerImage newDockerImage = new DockerImage("new-image"); final long wantedRestartGeneration = 2; final long currentRestartGeneration = 1; @@ -230,7 +230,7 @@ public class NodeAgentImplTest { } @Test - public void containerIsRestartedIfFlavorChanged() throws Exception { + public void containerIsRestartedIfFlavorChanged() { final long wantedRestartGeneration = 1; final long currentRestartGeneration = 1; ContainerNodeSpec.Builder specBuilder = nodeSpecBuilder @@ -268,7 +268,7 @@ public class NodeAgentImplTest { } @Test - public void noRestartIfOrchestratorSuspendFails() throws Exception { + public void noRestartIfOrchestratorSuspendFails() { final long wantedRestartGeneration = 2; final long currentRestartGeneration = 1; final ContainerNodeSpec nodeSpec = nodeSpecBuilder @@ -294,7 +294,7 @@ public class NodeAgentImplTest { } @Test - public void failedNodeRunningContainerShouldStillBeRunning() throws Exception { + public void failedNodeRunningContainerShouldStillBeRunning() { final long restartGeneration = 1; final long rebootGeneration = 0; final ContainerNodeSpec nodeSpec = nodeSpecBuilder @@ -324,7 +324,7 @@ public class NodeAgentImplTest { } @Test - public void readyNodeLeadsToNoAction() throws Exception { + public void readyNodeLeadsToNoAction() { final long restartGeneration = 1; final long rebootGeneration = 0; final ContainerNodeSpec nodeSpec = nodeSpecBuilder @@ -356,7 +356,7 @@ public class NodeAgentImplTest { } @Test - public void inactiveNodeRunningContainerShouldStillBeRunning() throws Exception { + public void inactiveNodeRunningContainerShouldStillBeRunning() { final long restartGeneration = 1; final long rebootGeneration = 0; @@ -390,7 +390,7 @@ public class NodeAgentImplTest { } @Test - public void reservedNodeDoesNotUpdateNodeRepoWithVersion() throws Exception { + public void reservedNodeDoesNotUpdateNodeRepoWithVersion() { final long restartGeneration = 1; final long rebootGeneration = 0; @@ -462,7 +462,7 @@ public class NodeAgentImplTest { } @Test - public void provisionedNodeIsMarkedAsDirty() throws Exception { + public void provisionedNodeIsMarkedAsDirty() { final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) .nodeState(Node.State.provisioned) @@ -497,7 +497,7 @@ public class NodeAgentImplTest { } @Test - public void resumeProgramRunsUntilSuccess() throws Exception { + public void resumeProgramRunsUntilSuccess() { final long restartGeneration = 1; final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) @@ -626,7 +626,7 @@ public class NodeAgentImplTest { } @Test - public void testGetRelevantMetricsForReadyNode() throws Exception { + public void testGetRelevantMetricsForReadyNode() { final ContainerNodeSpec nodeSpec = nodeSpecBuilder .nodeState(Node.State.ready) .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java index 397380461d6..75e33db97d8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java @@ -144,4 +144,21 @@ public class CommandLineTest { .mapToInt(i -> i) .sum()); } + + @Test + public void addTokensWithMultipleWhiteSpaces() { + terminal.expectCommand("iptables -L 2>&1"); + commandLine.addTokens("iptables -L").execute(); + + terminal.verifyAllCommandsExecuted(); + } + + @Test + public void addTokensWithSpecialCharacters() { + terminal.expectCommand("find . ! -name hei 2>&1"); + commandLine.addTokens("find . ! -name hei").execute(); + + terminal.verifyAllCommandsExecuted(); + } + }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java index f5a661c8a66..49a10e34fa6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.util; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.component.PathResolver; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/node-admin/vespa-node-admin.spec b/node-admin/vespa-node-admin.spec index 049e66bf22f..176fef4dcce 100644 --- a/node-admin/vespa-node-admin.spec +++ b/node-admin/vespa-node-admin.spec @@ -18,6 +18,7 @@ URL: http://vespa.ai Requires: bash Requires: java-1.8.0-openjdk-headless Requires: vespa-standalone-container +Requires: vespa-node-maintainer Conflicts: vespa @@ -32,6 +33,7 @@ cp node-admin/src/main/application/services.xml "$app_dir" declare -a jar_components=( node-admin/target/node-admin-jar-with-dependencies.jar + node-admin/target/node-admin-app/components/* docker-api/target/docker-api-jar-with-dependencies.jar ) for path in "${jar_components[@]}"; do @@ -41,6 +43,9 @@ done mkdir -p %buildroot%_prefix/libexec/vespa cp node-admin/src/main/sh/node-admin.sh %buildroot%_prefix/libexec/vespa +mkdir -p %buildroot%_prefix/libexec/vespa/node-admin +cp node-admin/scripts/maintenance.sh %buildroot%_prefix/libexec/vespa/node-admin + %clean rm -rf %buildroot diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java index de08bdbe107..c82f4406015 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java @@ -127,7 +127,7 @@ public class CoreCollector { installStatePath.ifPresent(installState -> { try { - data.put("yinst_state", readInstallState(installState)); + data.put("install_state", readInstallState(installState)); } catch (Exception e) { logger.log(Level.WARNING, "Failed to read install state", e); } diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java index 1e95ca15c3d..6c8388c3f28 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java @@ -142,7 +142,7 @@ public class Maintainer { Path coredumpsPath = Paths.get(getFieldOrFail(arguments, "coredumpsPath").asString()); Path doneCoredumpsPath = Paths.get(getFieldOrFail(arguments, "doneCoredumpsPath").asString()); Map<String, Object> attributesMap = parseMap(arguments); - Optional<Path> installStatePath = SlimeUtils.optionalString(arguments.field("yinstStatePath")).map(Paths::get); + Optional<Path> installStatePath = SlimeUtils.optionalString(arguments.field("installStatePath")).map(Paths::get); String feedEndpoint = getFieldOrFail(arguments, "feedEndpoint").asString(); try { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java index 869b53dc7a7..4b58c16bdc3 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollectorTest.java @@ -160,7 +160,7 @@ public class CoreCollectorTest { expectedData.put("bin_path", TEST_BIN_PATH.toString()); expectedData.put("backtrace", new ArrayList<>(GDB_BACKTRACE)); expectedData.put("backtrace_all_threads", new ArrayList<>(GDB_BACKTRACE)); - expectedData.put("yinst_state", new ArrayList<>(INSTALL_STATE)); + expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); expectedData.put("rpm_packages", new ArrayList<>(RPM_PACKAGES)); assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); } @@ -171,7 +171,7 @@ public class CoreCollectorTest { mockExec(new String[]{"cat", INSTALL_STATE_PATH.toString()}, String.join("\n", INSTALL_STATE)); Map<String, Object> expectedData = new HashMap<>(); - expectedData.put("yinst_state", new ArrayList<>(INSTALL_STATE)); + expectedData.put("install_state", new ArrayList<>(INSTALL_STATE)); assertEquals(expectedData, coreCollector.collect(TEST_CORE_PATH, Optional.of(INSTALL_STATE_PATH))); } diff --git a/node-maintainer/vespa-node-maintainer.spec b/node-maintainer/vespa-node-maintainer.spec new file mode 100644 index 00000000000..d9b04f23b66 --- /dev/null +++ b/node-maintainer/vespa-node-maintainer.spec @@ -0,0 +1,36 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# Force special prefix for Vespa +%define _prefix /opt/vespa + +# Hack to speed up jar packing for now. This does not affect the rpm size. +%define __jar_repack %{nil} + +Name: vespa-node-maintainer +Version: %version +Release: 1%{?dist} +BuildArch: noarch +Summary: Vespa Node Maintainer +Group: Applications/Databases +License: Commercial +URL: http://vespa.ai + +Requires: bash +Requires: java-1.8.0-openjdk-headless + +Conflicts: vespa + +%description +The Node Maintainer does various maintenance tasks on a node. + + +%install +mkdir -p %buildroot%_prefix/lib/jars +cp node-maintainer/target/node-maintainer-jar-with-dependencies.jar %buildroot%_prefix/lib/jars + +%clean +rm -rf %buildroot + +%files +%defattr(-,vespa,vespa,-) +%_prefix/* diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index c26e59a0b1a..830eb249ecb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -492,11 +492,7 @@ public class NodeRepository extends AbstractComponent { "Cannot make " + hostname + " available for new allocation, must be in state dirty, but was in " + node.state()); } - if (dynamicAllocationEnabled()) { - return removeRecursively(node, true); - } else { - return setReady(Collections.singletonList(node)); - } + return removeRecursively(node, true); } /** @@ -653,12 +649,4 @@ public class NodeRepository extends AbstractComponent { private Mutex lock(Node node) { return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } - - /* - * Temporary feature toggle to enable/disable dynamic docker allocation - * TODO: Remove when enabled in all zones - */ - public boolean dynamicAllocationEnabled() { - return curator.exists(Path.fromString("/provision/v1/dynamicDockerAllocation")); - } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 6a2376d748b..1d894f80eca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; @@ -26,7 +27,7 @@ public abstract class ApplicationMaintainer extends Maintainer { private final Deployer deployer; - private final Executor deploymentExecutor = Executors.newCachedThreadPool(); + private final Executor deploymentExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory("node repo application maintainer")); protected ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, JobControl jobControl) { super(nodeRepository, interval, jobControl); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 6089cfe64c9..7726311ab2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -248,11 +248,12 @@ public class NodeFailer extends Maintainer { // If the active node that we are trying to fail is of type host, we need to successfully fail all // the children nodes running on it before we fail the host boolean allTenantNodesFailedOutSuccessfully = true; + String reasonForChildFailure = "Failing due to parent host " + node.hostname() + " failure: " + reason; for (Node failingTenantNode : nodeRepository().getChildNodes(node.hostname())) { if (failingTenantNode.state() == Node.State.active) { - allTenantNodesFailedOutSuccessfully &= failActive(failingTenantNode, reason); + allTenantNodesFailedOutSuccessfully &= failActive(failingTenantNode, reasonForChildFailure); } else { - nodeRepository().fail(failingTenantNode.hostname(), Agent.system, reason); + nodeRepository().fail(failingTenantNode.hostname(), Agent.system, reasonForChildFailure); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index fae178adb87..b157f3d154f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -64,8 +64,7 @@ class GroupPreparer { prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); prioritizer.addReadyNodes(); - if (nodeRepository.dynamicAllocationEnabled()) - prioritizer.addNewDockerNodes(); + prioritizer.addNewDockerNodes(); // Allocate from the prioritized list NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index d9d7b4a5d12..2a140945f43 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -213,23 +213,23 @@ class NodeAllocation { * @return the final list of nodes */ List<Node> finalNodes(List<Node> surplusNodes) { - long currentRetired = nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); - long surplus = requestedNodes.surplusGiven(nodes.size()) - currentRetired; + int currentRetiredCount = (int) nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); + int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; - if (surplus > 0) { // retire until surplus is 0, prefer to retire higher indexes to minimize redistribution + if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0, prefer to retire higher indexes to minimize redistribution for (PrioritizableNode node : byDecreasingIndex(nodes)) { if ( ! node.node.allocation().get().membership().retired() && node.node.state().equals(Node.State.active)) { node.node = node.node.retire(Agent.application, clock.instant()); surplusNodes.add(node.node); // offer this node to other groups - if (--surplus == 0) break; + if (--deltaRetiredCount == 0) break; } } } - else if (surplus < 0) { // unretire until surplus is 0 + else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0 for (PrioritizableNode node : byIncreasingIndex(nodes)) { if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node.node)) { node.node = node.node.unretire(); - if (++surplus == 0) break; + if (++deltaRetiredCount == 0) break; } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 23a6e3a8b9a..dc3f4a64421 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -34,8 +34,8 @@ public interface NodeSpec { /** Returns whether the given node count is sufficient to fulfill this spec */ boolean fulfilledBy(int count); - /** Returns the amount the given count is above the minimum amount needed to fulfill this request */ - int surplusGiven(int count); + /** Returns the ideal number of nodes that should be retired to fulfill this spec */ + int idealRetiredCount(int acceptedCount, int currentRetiredCount); /** Returns a specification of a fraction of all the nodes of this. It is assumed the argument is a valid divisor. */ NodeSpec fraction(int divisor); @@ -97,7 +97,7 @@ public interface NodeSpec { public boolean saturatedBy(int count) { return fulfilledBy(count); } // min=max for count specs @Override - public int surplusGiven(int count) { return count - this.count; } + public int idealRetiredCount(int acceptedCount, int currentRetiredCount) { return acceptedCount - this.count; } @Override public NodeSpec fraction(int divisor) { return new CountNodeSpec(count/divisor, requestedFlavor); } @@ -152,7 +152,14 @@ public interface NodeSpec { public boolean saturatedBy(int count) { return false; } @Override - public int surplusGiven(int count) { return 0; } + public int idealRetiredCount(int acceptedCount, int currentRetiredCount) { + /* + * All nodes marked with wantToRetire get marked as retired just before this function is called, + * the job of this function is to throttle the retired count. If no nodes are marked as retired + * then continue this way, otherwise allow only 1 node to be retired + */ + return Math.min(1, currentRetiredCount); + } @Override public NodeSpec fraction(int divisor) { return this; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 4596b71300a..3b224fe2ba3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -55,11 +55,6 @@ public class MockNodeRepository extends NodeRepository { populate(); } - @Override - public boolean dynamicAllocationEnabled() { - return true; - } - private void populate() { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index fe3f12b105e..dce256c67f4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -61,15 +61,6 @@ public class NodeRepositoryTest { } @Test - public void featureToggleDynamicAllocationTest() { - NodeRepositoryTester tester = new NodeRepositoryTester(); - assertFalse(tester.nodeRepository().dynamicAllocationEnabled()); - - tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); - assertTrue(tester.nodeRepository().dynamicAllocationEnabled()); - } - - @Test public void only_allow_docker_containers_remove_in_ready() { NodeRepositoryTester tester = new NodeRepositoryTester(); tester.addNode("id1", "host1", "docker", NodeType.tenant); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java index 14c353e68f8..a0ee4900538 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -15,7 +15,6 @@ import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -62,7 +61,6 @@ public class DynamicDockerProvisioningTest { @Test public void relocate_nodes_from_headroom_hosts() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - enableDynamicAllocation(tester); tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); @@ -110,7 +108,6 @@ public class DynamicDockerProvisioningTest { @Test public void relocate_nodes_from_spare_hosts() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); @@ -157,7 +154,6 @@ public class DynamicDockerProvisioningTest { @Test public void new_docker_nodes_are_marked_as_headroom_violations() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - enableDynamicAllocation(tester); tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); @@ -213,7 +209,6 @@ public class DynamicDockerProvisioningTest { @Test public void only_preferred_container_is_moved_from_hosts_with_headroom_violations() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); - enableDynamicAllocation(tester); tester.makeReadyNodes(4, "host-medium", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); @@ -282,7 +277,6 @@ public class DynamicDockerProvisioningTest { @Test public void reloacte_failed_nodes() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); @@ -341,7 +335,6 @@ public class DynamicDockerProvisioningTest { @Test public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); tester.makeReadyNodes(2, "host-small", NodeType.host, 32); deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); @@ -369,7 +362,6 @@ public class DynamicDockerProvisioningTest { @Test(expected = OutOfCapacityException.class) public void multiple_groups_are_on_separate_parent_hosts() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); @@ -398,7 +390,6 @@ public class DynamicDockerProvisioningTest { } // Setup test - enableDynamicAllocation(tester); ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); @@ -438,7 +429,6 @@ public class DynamicDockerProvisioningTest { @Test public void non_prod_do_not_have_spares() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); tester.makeReadyNodes(3, "host-small", NodeType.host, 32); deployZoneApp(tester); Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); @@ -456,7 +446,6 @@ public class DynamicDockerProvisioningTest { @Test(expected = OutOfCapacityException.class) public void allocation_should_fail_when_host_is_not_active() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); - enableDynamicAllocation(tester); tester.makeProvisionedNodes(3, "host-small", NodeType.host, 32); deployZoneApp(tester); @@ -541,10 +530,6 @@ public class DynamicDockerProvisioningTest { return list; } - private void enableDynamicAllocation(ProvisioningTester tester) { - tester.getCurator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); - } - private boolean isInactiveOrRetired(Node node) { boolean isInactive = node.state().equals(Node.State.inactive); boolean isRetired = false; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 873193ac3b8..97fde2274f2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -11,15 +11,22 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.maintenance.JobControl; +import com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer; import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Before; import org.junit.Test; +import java.time.Duration; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * Tests provisioning by node type instead of by count and flavor @@ -28,28 +35,31 @@ import static org.junit.Assert.assertFalse; */ public class NodeTypeProvisioningTest { - @Test - public void proxy_deployment() { - ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + private final ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + private final ApplicationId application = tester.makeApplicationId(); // application using proxy nodes + private final Capacity capacity = Capacity.fromRequiredNodeType(NodeType.proxy); + private final ClusterSpec clusterSpec = ClusterSpec.request( + ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42")); + @Before + public void setup() { tester.makeReadyNodes( 1, "small", NodeType.proxy); tester.makeReadyNodes( 3, "small", NodeType.host); tester.makeReadyNodes( 5, "small", NodeType.tenant); tester.makeReadyNodes(10, "large", NodeType.proxy); tester.makeReadyNodes(20, "large", NodeType.host); tester.makeReadyNodes(40, "large", NodeType.tenant); + } - ApplicationId application = tester.makeApplicationId(); // application using proxy nodes - - + @Test + public void proxy_deployment() { { // Deploy List<HostSpec> hosts = deployProxies(application, tester); assertEquals("Reserved all proxies", 11, hosts.size()); tester.activate(application, new HashSet<>(hosts)); List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); assertEquals("Activated all proxies", 11, nodes.size()); - for (Node node : nodes) - assertEquals(NodeType.proxy, node.type()); } { // Redeploy with no changes @@ -83,14 +93,178 @@ public class NodeTypeProvisioningTest { } } + @Test + public void retire_proxy() { + MockDeployer deployer = new MockDeployer( + tester.provisioner(), + Collections.singletonMap( + application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1))); + RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, + tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())); + + { // Deploy + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals("Reserved all proxies", 11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals("Activated all proxies", 11, nodes.size()); + } + + Node nodeToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).get(5); + { // Pick out a node and retire it + tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true))); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that wantToRetire has been propagated + assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()) + .flatMap(Node::allocation) + .map(allocation -> allocation.membership().retired()) + .orElseThrow(RuntimeException::new)); + } + + { // Redeploying while the node is still retiring has no effect + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that the node is still marked as retired + assertTrue(tester.nodeRepository().getNode(nodeToRetire.hostname()) + .flatMap(Node::allocation) + .map(allocation -> allocation.membership().retired()) + .orElseThrow(RuntimeException::new)); + } + + { + tester.advanceTime(Duration.ofMinutes(11)); + retiredExpirer.run(); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(10, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(10, nodes.size()); + + // Verify that the node is now inactive + assertEquals(Node.State.inactive, tester.nodeRepository().getNode(nodeToRetire.hostname()) + .orElseThrow(RuntimeException::new).state()); + } + } + + @Test + public void retire_multiple_proxy_simultaneously() { + MockDeployer deployer = new MockDeployer( + tester.provisioner(), + Collections.singletonMap( + application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1))); + RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, + tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10), new JobControl(tester.nodeRepository().database())); + final int numNodesToRetire = 5; + + { // Deploy + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals("Reserved all proxies", 11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals("Activated all proxies", 11, nodes.size()); + } + + List<Node> nodesToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active) + .subList(3, 3 + numNodesToRetire); + String currentyRetiringHostname; + { + nodesToRetire.forEach(nodeToRetire -> + tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)))); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that wantToRetire has been propagated + List<Node> nodesCurrentlyRetiring = nodes.stream() + .filter(node -> node.allocation().get().membership().retired()) + .collect(Collectors.toList()); + assertEquals(1, nodesCurrentlyRetiring.size()); + + // The retiring node should be one of the nodes we marked for retirement + currentyRetiringHostname = nodesCurrentlyRetiring.get(0).hostname(); + assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(currentyRetiringHostname)).count() == 1); + } + + { // Redeploying while the node is still retiring has no effect + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(11, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(11, nodes.size()); + + // Verify that wantToRetire has been propagated + List<Node> nodesCurrentlyRetiring = nodes.stream() + .filter(node -> node.allocation().get().membership().retired()) + .collect(Collectors.toList()); + assertEquals(1, nodesCurrentlyRetiring.size()); + + // The node that started retiring is still the only one retiring + assertEquals(currentyRetiringHostname, nodesCurrentlyRetiring.get(0).hostname()); + } + + { + tester.advanceTime(Duration.ofMinutes(11)); + retiredExpirer.run(); + + List<HostSpec> hosts = deployProxies(application, tester); + assertEquals(10, hosts.size()); + tester.activate(application, new HashSet<>(hosts)); + List<Node> nodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active); + assertEquals(10, nodes.size()); + + // Verify the node we previously set to retire has finished retiring + assertEquals(Node.State.inactive, tester.nodeRepository().getNode(currentyRetiringHostname) + .orElseThrow(RuntimeException::new).state()); + + // Verify that a node is currently retiring + List<Node> nodesCurrentlyRetiring = nodes.stream() + .filter(node -> node.allocation().get().membership().retired()) + .collect(Collectors.toList()); + assertEquals(1, nodesCurrentlyRetiring.size()); + + // This node is different from the one that was retiring previously + String newRetiringHostname = nodesCurrentlyRetiring.get(0).hostname(); + assertNotEquals(currentyRetiringHostname, newRetiringHostname); + // ... but is one of the nodes that were put to wantToRetire earlier + assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(newRetiringHostname)).count() == 1); + } + + + for (int i = 0; i < 10; i++){ + tester.advanceTime(Duration.ofMinutes(11)); + retiredExpirer.run(); + List<HostSpec> hosts = deployProxies(application, tester); + tester.activate(application, new HashSet<>(hosts)); + } + + // After a long time, all currently active proxy nodes are not marked with wantToRetire or as retired + long numRetiredActiveProxyNodes = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).stream() + .filter(node -> !node.status().wantToRetire()) + .filter(node -> !node.allocation().get().membership().retired()) + .count(); + assertEquals(11 - numNodesToRetire, numRetiredActiveProxyNodes); + + // All the nodes that were marked with wantToRetire earlier are now inactive + assertEquals(nodesToRetire.stream().map(Node::hostname).collect(Collectors.toSet()), + tester.nodeRepository().getNodes(Node.State.inactive).stream().map(Node::hostname).collect(Collectors.toSet())); + } + private List<HostSpec> deployProxies(ApplicationId application, ProvisioningTester tester) { - return tester.prepare(application, - ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from("test"), - Version.fromString("6.42")), - Capacity.fromRequiredNodeType(NodeType.proxy), - 1); - + return tester.prepare(application, clusterSpec, capacity, 1); } } diff --git a/parent/pom.xml b/parent/pom.xml index ba03ae0d924..cbcc218b27d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -692,6 +692,11 @@ <artifactId>athenz-zts-java-client</artifactId> <version>${athenz.version}</version> </dependency> + <dependency> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock-standalone</artifactId> + <version>2.6.0</version> + </dependency> </dependencies> </dependencyManagement> @@ -702,7 +707,7 @@ <aries.util.version>1.0.0</aries.util.version> <asm-debug-all.version>5.0.3</asm-debug-all.version> <!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories --> - <athenz.version>1.7.28</athenz.version> + <athenz.version>1.7.43</athenz.version> <bouncycastle.version>1.58</bouncycastle.version> <commons-lang.version>2.6</commons-lang.version> <!-- WARNING: If you change curator version, you also need to update diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 9a4f094d32c..84adaa574ec 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -4,6 +4,7 @@ #include <vespa/document/select/parser.h> #include <vespa/document/base/documentid.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -17,6 +18,7 @@ LOG_SETUP(".dummypersistence"); using vespalib::make_string; using std::binary_search; using std::lower_bound; +using document::FixedBucketSpaces; namespace storage::spi::dummy { @@ -341,16 +343,18 @@ DummyPersistence::getPartitionStates() const BucketIdListResult -DummyPersistence::listBuckets(BucketSpace, PartitionId id) const +DummyPersistence::listBuckets(BucketSpace bucketSpace, PartitionId id) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "listBuckets(%u)", uint16_t(id)); vespalib::MonitorGuard lock(_monitor); BucketIdListResult::List list; - for (PartitionContent::const_iterator it = _content[id].begin(); - it != _content[id].end(); ++it) - { - list.push_back(it->first); + if (bucketSpace == FixedBucketSpaces::default_space()) { + for (PartitionContent::const_iterator it = _content[id].begin(); + it != _content[id].end(); ++it) + { + list.push_back(it->first); + } } return BucketIdListResult(list); } @@ -363,23 +367,30 @@ DummyPersistence::setModifiedBuckets(const BucketIdListResult::List& buckets) } BucketIdListResult -DummyPersistence::getModifiedBuckets(BucketSpace) const +DummyPersistence::getModifiedBuckets(BucketSpace bucketSpace) const { vespalib::MonitorGuard lock(_monitor); - return BucketIdListResult(_modifiedBuckets); + if (bucketSpace == FixedBucketSpaces::default_space()) { + return BucketIdListResult(_modifiedBuckets); + } else { + BucketIdListResult::List emptyList; + return BucketIdListResult(emptyList); + } } Result -DummyPersistence::setClusterState(BucketSpace, const ClusterState& c) +DummyPersistence::setClusterState(BucketSpace bucketSpace, const ClusterState& c) { vespalib::MonitorGuard lock(_monitor); - _clusterState.reset(new ClusterState(c)); - if (!_clusterState->nodeUp()) { - for (uint32_t i=0, n=_content.size(); i<n; ++i) { - for (PartitionContent::iterator it = _content[i].begin(); - it != _content[i].end(); ++it) - { - it->second->setActive(false); + if (bucketSpace == FixedBucketSpaces::default_space()) { + _clusterState.reset(new ClusterState(c)); + if (!_clusterState->nodeUp()) { + for (uint32_t i=0, n=_content.size(); i<n; ++i) { + for (PartitionContent::iterator it = _content[i].begin(); + it != _content[i].end(); ++it) + { + it->second->setActive(false); + } } } } @@ -394,6 +405,7 @@ DummyPersistence::setActiveState(const Bucket& b, LOG(debug, "setCurrentState(%s, %s)", b.toString().c_str(), newState == BucketInfo::ACTIVE ? "ACTIVE" : "INACTIVE"); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -407,6 +419,7 @@ BucketInfoResult DummyPersistence::getBucketInfo(const Bucket& b) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { LOG(debug, "getBucketInfo(%s) : (bucket not found)", @@ -430,6 +443,7 @@ DummyPersistence::put(const Bucket& b, Timestamp t, const Document::SP& doc, b.toString().c_str(), uint64_t(t), doc->getId().toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { return BucketInfoResult(Result::TRANSIENT_ERROR, "Bucket not found"); @@ -456,6 +470,7 @@ Result DummyPersistence::maintain(const Bucket& b, MaintenanceLevel) { + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); if (_simulateMaintainFailure) { BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -484,6 +499,7 @@ DummyPersistence::remove(const Bucket& b, b.toString().c_str(), uint64_t(t), did.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -511,6 +527,7 @@ DummyPersistence::get(const Bucket& b, LOG(debug, "get(%s, %s)", b.toString().c_str(), did.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { } else { @@ -538,6 +555,7 @@ DummyPersistence::createIterator( { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "createIterator(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); std::unique_ptr<document::select::Node> docSelection; if (!s.getDocumentSelection().getDocumentSelection().empty()) { docSelection.reset( @@ -712,6 +730,7 @@ DummyPersistence::createBucket(const Bucket& b, Context&) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "createBucket(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); if (_content[b.getPartition()].find(b) == _content[b.getPartition()].end()) { _content[b.getPartition()][b] = std::make_shared<BucketContent>(); @@ -727,6 +746,7 @@ DummyPersistence::deleteBucket(const Bucket& b, Context&) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "deleteBucket(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); if (_content[b.getPartition()][b].get()) { assert(!_content[b.getPartition()][b]->_inUse); @@ -746,6 +766,9 @@ DummyPersistence::split(const Bucket& source, source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); + assert(source.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(target1.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(target2.getBucketSpace() == FixedBucketSpaces::default_space()); createBucket(source, context); createBucket(target1, context); createBucket(target2, context); @@ -799,6 +822,9 @@ DummyPersistence::join(const Bucket& source1, const Bucket& source2, source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); + assert(source1.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(source2.getBucketSpace() == FixedBucketSpaces::default_space()); + assert(target.getBucketSpace() == FixedBucketSpaces::default_space()); createBucket(target, context); BucketContentGuard::UP targetGuard(acquireBucketWithLock(target)); assert(targetGuard.get()); @@ -833,6 +859,7 @@ DummyPersistence::revert(const Bucket& b, Timestamp t, Context&) LOG(debug, "revert(%s, %zu)", b.toString().c_str(), uint64_t(t)); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); BucketContentGuard::UP bc(acquireBucketWithLock(b)); if (!bc.get()) { @@ -883,6 +910,7 @@ DummyPersistence::dumpBucket(const Bucket& b) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(spam, "dumpBucket(%s)", b.toString().c_str()); + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); PartitionContent::const_iterator it(_content[b.getPartition()].find(b)); if (it == _content[b.getPartition()].end()) { @@ -902,6 +930,7 @@ bool DummyPersistence::isActive(const Bucket& b) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); LOG(spam, "isActive(%s)", b.toString().c_str()); PartitionContent::const_iterator it(_content[b.getPartition()].find(b)); @@ -919,6 +948,7 @@ BucketContentGuard::~BucketContentGuard() BucketContentGuard::UP DummyPersistence::acquireBucketWithLock(const Bucket& b) const { + assert(b.getBucketSpace() == FixedBucketSpaces::default_space()); vespalib::MonitorGuard lock(_monitor); DummyPersistence& ncp(const_cast<DummyPersistence&>(*this)); PartitionContent::iterator it(ncp._content[b.getPartition()].find(b)); diff --git a/protocols/getnodestate/distributor.json b/protocols/getnodestate/distributor.json index f4a0ab42243..970fd09f253 100644 --- a/protocols/getnodestate/distributor.json +++ b/protocols/getnodestate/distributor.json @@ -9,7 +9,23 @@ "latency-ms-sum": 10000, "count": 3 } - } + }, + "bucket-spaces" : [ + { + "name": "default", + "buckets": { + "total": 11, + "pending": 3 + } + }, + { + "name": "global", + "buckets": { + "total": 13, + "pending": 5 + } + } + ] }, { "node-index": 5, @@ -19,7 +35,12 @@ "latency-ms-sum": 25000, "count": 7 } - } + }, + "bucket-spaces" : [ + { + "name": "default" + } + ] } ] } diff --git a/protocols/getnodestate/host_info.json b/protocols/getnodestate/host_info.json index 5ef62d1a84f..305f279c847 100644 --- a/protocols/getnodestate/host_info.json +++ b/protocols/getnodestate/host_info.json @@ -48,20 +48,22 @@ "count": 16 } }, - "outstanding-merge-ops": { - "syncing": { - "buckets": 1 + "bucket-spaces": [ + { + "name": "default", + "buckets": { + "total": 11, + "pending": 3 + } }, - "copying-in": { - "buckets": 2 - }, - "moving-out": { - "buckets": 3 - }, - "copying-out": { - "buckets": 4 + { + "name": "global", + "buckets": { + "total": 13, + "pending": 5 + } } - } + ] }, { "node-index": 1, @@ -71,24 +73,13 @@ "latency-ms-sum": 17, "count": 18 } - }, - "outstanding-merge-ops": { - "syncing": { - "buckets": 1 - }, - "copying-in": { - "buckets": 2 - }, - "moving-out": { - "buckets": 3 - }, - "copying-out": { - "buckets": 4 + }, + "bucket-spaces": [ + { + "name": "default" } - } - + ] } - ] } } diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index a0f34072f06..f40502e14d1 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -16,7 +16,6 @@ LOG_SETUP("vespa-gen-testdocs"); typedef vespalib::hash_set<vespalib::string> StringSet; -typedef vespalib::hash_set<uint32_t> UIntSet; typedef std::vector<vespalib::string> StringArray; typedef std::shared_ptr<StringArray> StringArraySP; using namespace vespalib::alloc; @@ -41,6 +40,22 @@ prependBaseDir(const string &baseDir, return baseDir + "/" + file; } +std::vector<string> +splitArg(const string &arg) +{ + std::vector<string> argv; + string::size_type pos = 0; + for (;;) { + auto found = arg.find(',', pos); + if (found == string::npos) { + break; + } + argv.emplace_back(arg.substr(pos, found - pos)); + pos = found + 1; + } + argv.emplace_back(arg.substr(pos, string::npos)); + return argv; +} void shafile(const string &baseDir, @@ -149,26 +164,12 @@ protected: public: FieldGenerator(const string &name); - - virtual - ~FieldGenerator(); - - virtual void - setup(); - - virtual void - clear(); - - virtual void - deleteHistogram(const string &baseDir, - const string &name); - - virtual void - writeHistogram(const string &baseDir, - const string &name); - - virtual void - generate(vespalib::asciistream &doc, uint32_t id) = 0; + virtual ~FieldGenerator(); + virtual void setup(); + virtual void generateXML(vespalib::asciistream &doc, uint32_t id); + virtual void generateJSON(vespalib::asciistream &doc, uint32_t id); + virtual void generateValue(vespalib::asciistream &doc, uint32_t id); + virtual bool isString() const { return true; } }; @@ -182,45 +183,109 @@ FieldGenerator::~FieldGenerator() { } - void FieldGenerator::setup() { } - void -FieldGenerator::clear() +FieldGenerator::generateXML(vespalib::asciistream &doc, uint32_t id) { + doc << " <" << _name << ">"; + generateValue(doc, id); + doc << "</" << _name << ">\n"; } +void +FieldGenerator::generateJSON(vespalib::asciistream &doc, uint32_t id) +{ + doc << "\"" << _name << "\": "; + bool needQuote = isString(); + if (needQuote) { + doc << "\""; + } + generateValue(doc, id); + if (needQuote) { + doc << "\""; + } +} void -FieldGenerator::deleteHistogram(const string &baseDir, - const string &name) +FieldGenerator::generateValue(vespalib::asciistream &, uint32_t) +{ +} + +class ConstTextFieldGenerator : public FieldGenerator { - (void) baseDir; - (void) name; + string _value; +public: + ConstTextFieldGenerator(std::vector<string> argv); + virtual ~ConstTextFieldGenerator() override; + virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; +}; + +ConstTextFieldGenerator::ConstTextFieldGenerator(std::vector<string> argv) + : FieldGenerator(argv[0]), + _value() +{ + if (argv.size() > 1) { + _value = argv[1]; + } } +ConstTextFieldGenerator::~ConstTextFieldGenerator() = default; void -FieldGenerator::writeHistogram(const string &baseDir, - const string &name) +ConstTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t) +{ + doc << _value; +} + +class PrefixTextFieldGenerator : public FieldGenerator { - (void) baseDir; - (void) name; + string _prefix; + uint32_t _mod; + uint32_t _div; +public: + PrefixTextFieldGenerator(std::vector<string> argv); + virtual ~PrefixTextFieldGenerator() override; + virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; +}; + +PrefixTextFieldGenerator::PrefixTextFieldGenerator(std::vector<string> argv) + : FieldGenerator(argv[0]), + _prefix(), + _mod(std::numeric_limits<uint32_t>::max()), + _div(1u) +{ + if (argv.size() > 1) { + _prefix = argv[1]; + if (argv.size() > 2) { + if (!argv[2].empty()) { + _mod = atol(argv[2].c_str()); + } + if (argv.size() > 3) { + if (!argv[3].empty()) { + _div = atol(argv[3].c_str()); + } + } + } + } } +PrefixTextFieldGenerator::~PrefixTextFieldGenerator() = default; + +void +PrefixTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) +{ + doc << _prefix << ((id / _div) % _mod); +} class RandTextFieldGenerator : public FieldGenerator { search::Rand48 &_rnd; uint32_t _numWords; StringArray _strings; - std::vector<uint32_t> _histogram; - UIntSet _wnums; - uint32_t _colls; uint32_t _minFill; uint32_t _randFill; @@ -230,24 +295,9 @@ public: uint32_t numWords, uint32_t minFill, uint32_t maxFill); - - virtual - ~RandTextFieldGenerator(); - - virtual void - setup() override; - - virtual void - clear() override; - - virtual void - deleteHistogram(const string &baseDir, const string &name) override; - - virtual void - writeHistogram(const string &baseDir, const string &name) override; - - virtual void - generate(vespalib::asciistream &doc, uint32_t id) override; + virtual ~RandTextFieldGenerator(); + virtual void setup() override; + virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; }; @@ -260,9 +310,6 @@ RandTextFieldGenerator::RandTextFieldGenerator(const string &name, _rnd(rnd), _numWords(numWords), _strings(), - _histogram(), - _wnums(), - _colls(0u), _minFill(minFill), _randFill(randFill) { @@ -282,60 +329,12 @@ RandTextFieldGenerator::setup() "generating dictionary for field %s (%u words)", _name.c_str(), _numWords); StringGenerator(_rnd).rand_unique_array(_strings, 5, 10, _numWords); - _histogram.resize(_numWords); -} - - -void -RandTextFieldGenerator::clear() -{ - typedef std::vector<uint32_t>::iterator HI; - for (HI i(_histogram.begin()), ie(_histogram.end()); i != ie; ++i) { - *i = 0; - } - _colls = 0; -} - - -void -RandTextFieldGenerator::deleteHistogram(const string &baseDir, - const string &name) -{ - string fname(prependBaseDir(baseDir, name) + "-" + _name); - FastOS_File::Delete(fname.c_str()); -} - - -void -RandTextFieldGenerator::writeHistogram(const string &baseDir, - const string &name) -{ - LOG(info, "%u word collisions for field %s", _colls, _name.c_str()); - string fname(name + "-" + _name); - string fullName(prependBaseDir(baseDir, fname)); - LOG(info, "Writing histogram %s", fname.c_str()); - Fast_BufferedFile f(new FastOS_File); - f.WriteOpen(fullName.c_str()); - uint32_t numWords = _strings.size(); - assert(numWords == _histogram.size()); - for (uint32_t wNum = 0; wNum < numWords; ++wNum) { - f.WriteString(_strings[wNum].c_str()); - f.WriteString(" "); - f.addNum(_histogram[wNum], 0, ' '); - f.WriteString("\n"); - } - f.Flush(); - f.Close(); - shafile(baseDir, fname); } void -RandTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) +RandTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t) { - (void) id; - doc << " <" << _name << ">"; - _wnums.clear(); uint32_t gLen = _minFill + _rnd.lrand48() % (_randFill + 1); bool first = true; for (uint32_t i = 0; i < gLen; ++i) { @@ -343,18 +342,12 @@ RandTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) doc << " "; first = false; uint32_t wNum = _rnd.lrand48() % _strings.size(); - if (_wnums.insert(wNum).second) - _histogram[wNum]++; - else - ++_colls; const string &s(_strings[wNum]); assert(s.size() > 0); doc << s; } - doc << "</" << _name << ">\n"; } - class ModTextFieldGenerator : public FieldGenerator { search::Rand48 &_rnd; @@ -364,18 +357,8 @@ public: ModTextFieldGenerator(const string &name, search::Rand48 &rnd, const std::vector<uint32_t> &mods); - - virtual - ~ModTextFieldGenerator(); - - virtual void - clear() override; - - virtual void - writeHistogram(const string &name); - - virtual void - generate(vespalib::asciistream &doc, uint32_t id) override; + virtual ~ModTextFieldGenerator(); + virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; }; @@ -395,23 +378,9 @@ ModTextFieldGenerator::~ModTextFieldGenerator() void -ModTextFieldGenerator::clear() -{ -} - - -void -ModTextFieldGenerator::writeHistogram(const string &name) -{ - (void) name; -} - - -void -ModTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) +ModTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) { typedef std::vector<uint32_t>::const_iterator MI; - doc << " <" << _name << ">"; bool first = true; for (MI mi(_mods.begin()), me(_mods.end()); mi != me; ++mi) { uint32_t m = *mi; @@ -420,7 +389,6 @@ ModTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) first = false; doc << "w" << m << "w" << (id % m); } - doc << "</" << _name << ">\n"; } @@ -428,18 +396,8 @@ class IdTextFieldGenerator : public FieldGenerator { public: IdTextFieldGenerator(const string &name); - - virtual - ~IdTextFieldGenerator(); - - virtual void - clear() override; - - virtual void - writeHistogram(const string &name); - - virtual void - generate(vespalib::asciistream &doc, uint32_t id) override; + virtual ~IdTextFieldGenerator(); + virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; }; @@ -455,24 +413,9 @@ IdTextFieldGenerator::~IdTextFieldGenerator() void -IdTextFieldGenerator::clear() -{ -} - - -void -IdTextFieldGenerator::writeHistogram(const string &name) -{ - (void) name; -} - - -void -IdTextFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) +IdTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) { - doc << " <" << _name << ">"; doc << id; - doc << "</" << _name << ">\n"; } @@ -487,18 +430,9 @@ public: search::Rand48 &rnd, uint32_t low, uint32_t count); - - virtual - ~RandIntFieldGenerator(); - - virtual void - clear() override; - - virtual void - writeHistogram(const string &name); - - virtual void - generate(vespalib::asciistream &doc, uint32_t id) override; + virtual ~RandIntFieldGenerator(); + virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; + virtual bool isString() const override { return false; } }; @@ -521,26 +455,10 @@ RandIntFieldGenerator::~RandIntFieldGenerator() void -RandIntFieldGenerator::clear() -{ -} - - -void -RandIntFieldGenerator::writeHistogram(const string &name) -{ - (void) name; -} - - -void -RandIntFieldGenerator::generate(vespalib::asciistream &doc, uint32_t id) +RandIntFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t) { - (void) id; - doc << " <" << _name << ">"; uint32_t r = _low + _rnd.lrand48() % _count; doc << r; - doc << "</" << _name << ">\n"; } class DocumentGenerator @@ -557,28 +475,13 @@ public: DocumentGenerator(const string &docType, const string &idPrefix, const FieldVec &fields); - ~DocumentGenerator(); - - void - clear(); - - void - deleteHistogram(const string &baseDir, - const string &name); - - void - writeHistogram(const string &baseDir, - const string &name); - - void - generate(uint32_t id); - - void - generate(uint32_t docMin, uint32_t docCount, - const string &baseDir, - const string &feedFileName, - bool headers); + void generateXML(uint32_t id); + void generateJSON(uint32_t id); + void generate(uint32_t docMin, uint32_t docIdLimit, + const string &baseDir, + const string &feedFileName, + bool headers, bool json); }; @@ -609,79 +512,73 @@ DocumentGenerator::setup() void -DocumentGenerator::clear() -{ - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->clear(); - } -} - - -void -DocumentGenerator::generate(uint32_t id) +DocumentGenerator::generateXML(uint32_t id) { _doc.clear(); - _doc << "<document documenttype=\"" << _docType << "\" documentid= \"" << + _doc << "<document documenttype=\"" << _docType << "\" documentid=\"" << _idPrefix << id << "\">\n"; - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->generate(_doc, id); + for (const auto &field : _fields) { + field->generateXML(_doc, id); } _doc << "</document>\n"; } - void -DocumentGenerator::deleteHistogram(const string &baseDir, - const string &name) +DocumentGenerator::generateJSON(uint32_t id) { - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->deleteHistogram(baseDir, name); - } -} - -void -DocumentGenerator::writeHistogram(const string &baseDir, - const string &name) -{ - typedef FieldVec::const_iterator FI; - for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { - (*i)->writeHistogram(baseDir, name); + _doc.clear(); + _doc << " { \"put\": \"" << _idPrefix << id << "\",\n \"fields\": {"; + bool first = true; + for (const auto &field : _fields) { + if (!first) { + _doc << ","; + } + first = false; + _doc << "\n "; + field->generateJSON(_doc, id); } + _doc << "\n }\n }"; } - + void -DocumentGenerator::generate(uint32_t docMin, uint32_t docCount, +DocumentGenerator::generate(uint32_t docMin, uint32_t docIdLimit, const string &baseDir, const string &feedFileName, - bool headers) + bool headers, bool json) { string fullName(prependBaseDir(baseDir, feedFileName)); FastOS_File::Delete(fullName.c_str()); - string histname(feedFileName + ".histogram"); - deleteHistogram(baseDir, histname); Fast_BufferedFile f(new FastOS_File); f.WriteOpen(fullName.c_str()); - clear(); - if (headers) { - f.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"); - f.WriteString("<vespafeed>\n"); - } - uint32_t docLim = docMin + docCount; - for (uint32_t id = docMin; id < docLim; ++id) { - generate(id); - f.WriteBuf(_doc.c_str(), _doc.size()); - } - if (headers) { - f.WriteString("</vespafeed>\n"); + if (json) { + bool first = true; + f.WriteString("[\n"); + for (uint32_t id = docMin; id < docIdLimit; ++id) { + if (!first) { + f.WriteString(",\n"); + } + first = false; + generateJSON(id); + f.WriteBuf(_doc.c_str(), _doc.size()); + } + f.WriteString("\n]\n"); + } else { + if (headers) { + f.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"); + f.WriteString("<vespafeed>\n"); + } + for (uint32_t id = docMin; id < docIdLimit; ++id) { + generateXML(id); + f.WriteBuf(_doc.c_str(), _doc.size()); + } + if (headers) { + f.WriteString("</vespafeed>\n"); + } } f.Flush(); f.Close(); LOG(info, "Calculating sha256 for %s", feedFileName.c_str()); shafile(baseDir, feedFileName); - writeHistogram(baseDir, histname); } @@ -725,6 +622,7 @@ class GenTestDocsApp : public SubApp search::Rand48 _rnd; string _outFile; bool _headers; + bool _json; public: GenTestDocsApp(FastOS_Application &app) @@ -740,7 +638,8 @@ public: _mods(), _rnd(), _outFile(), - _headers(false) + _headers(false), + _json(false) { _mods.push_back(2); _mods.push_back(3); @@ -775,6 +674,8 @@ GenTestDocsApp::usage(bool showHeader) cerr << "vespa-gen-testdocs gentestdocs\n" " [--basedir basedir]\n" + " [--consttextfield name]\n" + " [--prefixtextfield name]\n" " [--randtextfield name]\n" " [--modtextfield name]\n" " [--idtextfield name]\n" @@ -784,6 +685,7 @@ GenTestDocsApp::usage(bool showHeader) " [--numwords numWords]\n" " [--doctype docType]\n" " [--headers]\n" + " [--json]\n" " outFile\n"; } @@ -795,6 +697,8 @@ GenTestDocsApp::getOptions() int longopt_index = 0; static struct option longopts[] = { { "basedir", 1, NULL, 0 }, + { "consttextfield", 1, NULL, 0 }, + { "prefixtextfield", 1, NULL, 0 }, { "randtextfield", 1, NULL, 0 }, { "modtextfield", 1, NULL, 0 }, { "idtextfield", 1, NULL, 0 }, @@ -804,10 +708,13 @@ GenTestDocsApp::getOptions() { "numwords", 1, NULL, 0 }, { "doctype", 1, NULL, 0 }, { "headers", 0, NULL, 0 }, + { "json", 0, NULL, 0 }, { NULL, 0, NULL, 0 } }; enum longopts_enum { LONGOPT_BASEDIR, + LONGOPT_CONSTTEXTFIELD, + LONGOPT_PREFIXTEXTFIELD, LONGOPT_RANDTEXTFIELD, LONGOPT_MODTEXTFIELD, LONGOPT_IDTEXTFIELD, @@ -816,7 +723,8 @@ GenTestDocsApp::getOptions() LONGOPT_MINDOCID, LONGOPT_NUMWORDS, LONGOPT_DOCTYPE, - LONGOPT_HEADERS + LONGOPT_HEADERS, + LONGOPT_JSON }; int optIndex = 2; while ((c = _app.GetOptLong("v", @@ -831,6 +739,12 @@ GenTestDocsApp::getOptions() case LONGOPT_BASEDIR: _baseDir = optArgument; break; + case LONGOPT_CONSTTEXTFIELD: + _fields.emplace_back(std::make_shared<ConstTextFieldGenerator>(splitArg(optArgument))); + break; + case LONGOPT_PREFIXTEXTFIELD: + _fields.emplace_back(std::make_shared<PrefixTextFieldGenerator>(splitArg(optArgument))); + break; case LONGOPT_RANDTEXTFIELD: g.reset(new RandTextFieldGenerator(optArgument, _rnd, @@ -856,7 +770,6 @@ GenTestDocsApp::getOptions() 100000)); _fields.push_back(g); break; - break; case LONGOPT_DOCIDLIMIT: _docIdLimit = atoi(optArgument); break; @@ -872,6 +785,9 @@ GenTestDocsApp::getOptions() case LONGOPT_HEADERS: _headers = true; break; + case LONGOPT_JSON: + _json = true; + break; default: if (optArgument != NULL) { LOG(error, @@ -911,7 +827,7 @@ GenTestDocsApp::run() idPrefix, _fields); LOG(info, "generating %s", _outFile.c_str()); - dg.generate(_minDocId, _docIdLimit, _baseDir, _outFile, _headers); + dg.generate(_minDocId, _docIdLimit, _baseDir, _outFile, _headers, _json); LOG(info, "done"); return 0; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 98642c6edf6..151b1e1596b 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -869,6 +869,8 @@ FastS_FNET_Search::CheckCoverage() uint64_t activeDocs = 0; uint64_t soonActiveDocs = 0; uint32_t degradedReason = 0; + uint16_t nodesQueried = 0; + uint16_t nodesReplied = 0; size_t cntNone(0); for (const FastS_FNET_SearchNode & node : _nodes) { @@ -877,14 +879,17 @@ FastS_FNET_Search::CheckCoverage() activeDocs += node._qresult->_activeDocs; soonActiveDocs += node._qresult->_soonActiveDocs; degradedReason |= node._qresult->_coverageDegradeReason; + nodesQueried += node._qresult->getNodesQueried(); + nodesReplied += node._qresult->getNodesReplied(); } else { + nodesQueried++; cntNone++; } } if ((cntNone > 0) && (cntNone != _nodes.size())) { activeDocs += cntNone * activeDocs/(_nodes.size() - cntNone); } - _util.SetCoverage(covDocs, activeDocs, soonActiveDocs, degradedReason, _nodes.size(), _nodes.size() - cntNone); + _util.SetCoverage(covDocs, activeDocs, soonActiveDocs, degradedReason, nodesQueried, nodesReplied); } @@ -1151,7 +1156,7 @@ FastS_FNET_Search::setupQueryPacket(uint32_t hitsPerNode, uint32_t qflags, qx._features = search::fs4transport::QF_PARSEDQUERY | search::fs4transport::QF_RANKP; qx._offset = _util.GetAlignedSearchOffset(); qx._maxhits = hitsPerNode; // capped maxhits - qx._qflags = qflags; // filtered query flags + qx.setQueryFlags(qflags); qx.setTimeout(_queryArgs->getTimeLeft()); qx.setRanking(_queryArgs->ranking); diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp index ad4cf02f4e8..35ca7bc1dd9 100644 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ b/searchlib/src/tests/common/packets/packets_test.cpp @@ -352,8 +352,8 @@ TEST("testQueryResultX") { EXPECT_EQUAL(2u, ptr->_totNumDocs); EXPECT_EQUAL((search::HitRank)3, ptr->_maxRank); EXPECT_EQUAL(4u, ptr->getDistributionKey()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 0u, ptr->getNodesQueried()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 0u, ptr->getNodesReplied()); + EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 1u, ptr->getNodesQueried()); + EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 1u, ptr->getNodesReplied()); EXPECT_EQUAL(6u, ptr->_coverageDocs); EXPECT_EQUAL(7u, ptr->_activeDocs); EXPECT_EQUAL(8u, ptr->_soonActiveDocs); @@ -398,7 +398,7 @@ createAndFill_QUERYX() EXPECT_EQUAL(0l, src->getTimeout()); src->setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), src->getTimeout()); - src->_qflags = 5u; + src->setQueryFlags(5u); src->setRanking("seven"); src->_numStackItems = 14u; src->_propsVector.resize(2); @@ -419,7 +419,7 @@ verifyQueryX(FS4Packet_QUERYX & queryX, uint32_t features) EXPECT_EQUAL(2u, queryX._offset); EXPECT_EQUAL(3u, queryX._maxhits); EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), queryX.getTimeout()); - EXPECT_EQUAL(0x5u, queryX._qflags); + EXPECT_EQUAL(0x1u, queryX.getQueryFlags()); //Filtered if (queryX._features & QF_RANKP) { EXPECT_EQUAL("seven", queryX._ranking); } else { diff --git a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp index 3e8be3a99fc..a517890620c 100644 --- a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp +++ b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp @@ -54,7 +54,7 @@ Test::convertToRequest() src._offset = 2u; src._maxhits = 3u; src.setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - src._qflags = 5u; + src.setQueryFlags(5u); src._features |= QF_RANKP; src.setRanking("seven"); src._features |= QF_PROPERTIES; @@ -92,7 +92,7 @@ Test::convertToRequest() EXPECT_EQUAL(dst.offset, 2u); EXPECT_EQUAL(dst.maxhits, 3u); EXPECT_EQUAL((dst.getTimeOfDoom() - dst.getStartTime()).ms(), 4u); - EXPECT_EQUAL(dst.queryFlags, 5u); + EXPECT_EQUAL(dst.queryFlags, 1u); //Filtered EXPECT_EQUAL(vespalib::string("seven"), dst.ranking); EXPECT_EQUAL(dst.propertiesMap.size(), 2u); EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k1").get(), std::string("p1v1")); diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 4bb11acaabb..5004ba80913 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -906,8 +906,8 @@ FS4Packet_QUERYRESULTX::AllocateHits(uint32_t cnt) FS4Packet_QUERYRESULTX::FS4Packet_QUERYRESULTX() : FS4Packet(), _distributionKey(0), - _nodesQueried(0), - _nodesReplied(0), + _nodesQueried(1), + _nodesReplied(1), _features(QRF_COVERAGE | QRF_EXTENDED_COVERAGE), _offset(0), _numDocs(0), @@ -1172,10 +1172,10 @@ FS4Packet_QUERYRESULTX::toString(uint32_t indent) const FS4Packet_QUERYX::FS4Packet_QUERYX() : FS4Packet(), _timeout(0), + _qflags(0), _features(0), _offset(0), _maxhits(0), - _qflags(0), _ranking(), _propsVector(), _sortSpec(), diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index a876400e503..87e2bd998b1 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -42,6 +42,12 @@ enum fnet_feature_masks { GDF_RESCLASSNAME | GDF_PROPERTIES | GDF_FLAGS), + ACTIVE_QUERY_FLAGS = (QFLAG_EXTENDED_COVERAGE | + QFLAG_COVERAGE_NODES | + QFLAG_ESTIMATE | + QFLAG_DROP_SORTDATA | + QFLAG_NO_RESULTCACHE | + QFLAG_DUMP_FEATURES), FNET_MQF_SUPPORTED_MASK = (MQF_QFLAGS), @@ -441,12 +447,12 @@ private: FS4Packet_QUERYX& operator=(const FS4Packet_QUERYX &); uint32_t _timeout; + uint32_t _qflags; public: uint32_t _features; // see query_features uint32_t _offset; uint32_t _maxhits; - uint32_t _qflags; string _ranking; // if QF_RANKP PropsVector _propsVector; // if QF_PROPERTIES string _sortSpec; // if QF_SORTSPEC @@ -457,6 +463,7 @@ public: uint32_t _numStackItems; // if QF_PARSEDQUERY string _stackDump; // if QF_PARSEDQUERY + void setQueryFlags(uint32_t qflags) { _qflags = ACTIVE_QUERY_FLAGS & qflags; } void setRanking(const vespalib::stringref &ranking) { _ranking = ranking; } void setSortSpec(const vespalib::stringref &spec) { _sortSpec = spec; } void setGroupSpec(const vespalib::stringref &spec) { _groupSpec = spec; } @@ -465,6 +472,7 @@ public: void setStackDump(const vespalib::stringref &buf) { _stackDump = buf; } void setTimeout(const fastos::TimeStamp & timeout); fastos::TimeStamp getTimeout() const; + uint32_t getQueryFlags() const { return _qflags; } explicit FS4Packet_QUERYX(); ~FS4Packet_QUERYX(); diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h index b6a925240c9..55cb44a34d0 100644 --- a/searchlib/src/vespa/searchlib/common/transport.h +++ b/searchlib/src/vespa/searchlib/common/transport.h @@ -12,6 +12,7 @@ namespace search::fs4transport { * are as follows: * <ul> * <li><b>QFLAG_EXTENDED_COVERAGE</b>: Indicates that the it is able to receive extended coverage information.</li> + * <li><b>QFLAG_COVERAGE_NODES</b>: Indicate that it is able to handle nodes information.</li> * <li><b>QFLAG_ESTIMATE</b>: Indicates that the query is performed to get * an estimate of the total number of hits</li> * <li><b>QFLAG_DUMP_FEATURES</b>: Dump detailed ranking information. Note that diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp index 2ebbb9bf89c..e6fb37223d6 100644 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp +++ b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp @@ -55,7 +55,7 @@ PacketConverter::toSearchRequest(const QUERYX &packet, SearchRequest &request) request.offset = packet._offset; request.maxhits = packet._maxhits; request.setTimeout(packet.getTimeout()); - request.queryFlags = packet._qflags; + request.queryFlags = packet.getQueryFlags(); request.ranking = packet._ranking; for (uint32_t i = 0; i < packet._propsVector.size(); ++i) { diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigInstallVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigInstallVariables.scala new file mode 100644 index 00000000000..8cfa01937c9 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigInstallVariables.scala @@ -0,0 +1,91 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import java.util.Optional + +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer + +import scala.language.implicitConversions +import scala.util.Try + +/** + * @author Tony Vaagenes + */ +class CloudConfigInstallVariables extends CloudConfigOptions { + import CloudConfigInstallVariables._ + + override val rpcPort = optionalInstallVar[Integer]("port_configserver_rpc", "services") + override val allConfigServers = installVar("addr_configserver", "services") withDefault Array[ConfigServer]() + override val multiTenant = optionalInstallVar[java.lang.Boolean]("multitenant") + + override val zookeeperBarrierTimeout = optionalInstallVar[java.lang.Long]("zookeeper_barrier_timeout") + override val sessionLifeTimeSecs = optionalInstallVar[java.lang.Long]("session_lifetime") + override val configModelPluginDirs = installVar("config_model_plugin_dirs") withDefault Array[String]() + override val zookeeperClientPort = optionalInstallVar[Integer]("zookeeper_clientPort") + override val zookeeperQuorumPort = optionalInstallVar[Integer]("zookeeper_quoromPort") + override val zookeeperElectionPort = optionalInstallVar[Integer]("zookeeper_electionPort") + override val payloadCompressionType = optionalInstallVar[java.lang.String]("payload_compression_type") + override val environment = optionalInstallVar[java.lang.String]("environment") + override val region = optionalInstallVar[java.lang.String]("region") + override val system = optionalInstallVar[java.lang.String]("system") + override val defaultFlavor = optionalInstallVar[java.lang.String]("default_flavor") + override val defaultAdminFlavor = optionalInstallVar[java.lang.String]("default_admin_flavor") + override val defaultContainerFlavor = optionalInstallVar[java.lang.String]("default_container_flavor") + override val defaultContentFlavor = optionalInstallVar[java.lang.String]("default_content_flavor") + override val useVespaVersionInRequest = optionalInstallVar[java.lang.Boolean]("use_vespa_version_in_request") + override val hostedVespa = optionalInstallVar[java.lang.Boolean]("hosted_vespa") + override val numParallelTenantLoaders = optionalInstallVar[java.lang.Integer]("num_parallel_tenant_loaders") + override val dockerRegistry = optionalInstallVar[java.lang.String]("docker_registry") + override val dockerVespaBaseImage = optionalInstallVar[java.lang.String]("docker_vespa_base_image") + override val loadBalancerAddress = optionalInstallVar[java.lang.String]("load_balancer_address") +} + +object CloudConfigInstallVariables { + private class InstallVariable(installPkg:String, name: String) { + val value = Environment.optionalInstallVariable(installPkg + "." + name) + + def withDefault[T](defaultValue: T)(implicit c: Converter[T]) : T = { + value map { implicitly[Converter[T]].convert } getOrElse defaultValue + } + } + + private def installVar(setting:String, installPkg: String = "cloudconfig_server") = new InstallVariable(installPkg, setting) + + private def optionalInstallVar[T](setting:String, installPkg: String = "cloudconfig_server")(implicit c: Converter[T]): Optional[T] = { + Environment.optionalInstallVariable(installPkg + "." + setting) map ( c.convert ) + } + + implicit val configServerConverter: Converter[Array[ConfigServer]] = new Converter[Array[ConfigServer]] { + override def convert(s: String) = { + s split "[, ]" filter { !_.isEmpty } map { toConfigServer } + } + } + + implicit val stringArrayConverter: Converter[Array[String]] = new Converter[Array[String]] { + override def convert(s: String) = { + s split "[, ]" filter { !_.isEmpty } + } + } + + private def toConfigServer(hostPort: String): ConfigServer = Try { + val (host, portStr) = splitFirst(hostPort, ':') + val port = portStr map { _.toInt } + new ConfigServer(host, port) + }.getOrElse(throw new IllegalArgumentException(s"Invalid config server '$hostPort'")) + + private def splitFirst(string: String, separator: Character): (String, Option[String]) = { + val (beginning, endWithSeparator) = string span { _ != separator } + (beginning, tailOption(endWithSeparator)) + } + + def tailOption(s: String) = { + if (s.isEmpty) None + else Some(s.tail) + } + + implicit def toJavaOptional[U <% V, V](option: Option[U]): Optional[V] = option match { + case Some(u) => Optional.of(u: V) + case None => Optional.empty() + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala deleted file mode 100644 index 41e6b66b986..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import java.util.Optional - -import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions -import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer - -import scala.language.implicitConversions -import scala.util.Try - -/** - * @author Tony Vaagenes - */ -class CloudConfigYinstVariables extends CloudConfigOptions { - import CloudConfigYinstVariables._ - - override val rpcPort = optionalYinstVar[Integer]("port_configserver_rpc", "services") - override val allConfigServers = yinstVar("addr_configserver", "services") withDefault Array[ConfigServer]() - override val multiTenant = optionalYinstVar[java.lang.Boolean]("multitenant") - - override val zookeeperBarrierTimeout = optionalYinstVar[java.lang.Long]("zookeeper_barrier_timeout") - override val sessionLifeTimeSecs = optionalYinstVar[java.lang.Long]("session_lifetime") - override val configModelPluginDirs = yinstVar("config_model_plugin_dirs") withDefault Array[String]() - override val zookeeperClientPort = optionalYinstVar[Integer]("zookeeper_clientPort") - override val zookeeperQuorumPort = optionalYinstVar[Integer]("zookeeper_quoromPort") - override val zookeeperElectionPort = optionalYinstVar[Integer]("zookeeper_electionPort") - override val payloadCompressionType = optionalYinstVar[java.lang.String]("payload_compression_type") - override val environment = optionalYinstVar[java.lang.String]("environment") - override val region = optionalYinstVar[java.lang.String]("region") - override val system = optionalYinstVar[java.lang.String]("system") - override val defaultFlavor = optionalYinstVar[java.lang.String]("default_flavor") - override val defaultAdminFlavor = optionalYinstVar[java.lang.String]("default_admin_flavor") - override val defaultContainerFlavor = optionalYinstVar[java.lang.String]("default_container_flavor") - override val defaultContentFlavor = optionalYinstVar[java.lang.String]("default_content_flavor") - override val useVespaVersionInRequest = optionalYinstVar[java.lang.Boolean]("use_vespa_version_in_request") - override val hostedVespa = optionalYinstVar[java.lang.Boolean]("hosted_vespa") - override val numParallelTenantLoaders = optionalYinstVar[java.lang.Integer]("num_parallel_tenant_loaders") - override val dockerRegistry = optionalYinstVar[java.lang.String]("docker_registry") - override val dockerVespaBaseImage = optionalYinstVar[java.lang.String]("docker_vespa_base_image") - override val loadBalancerAddress = optionalYinstVar[java.lang.String]("load_balancer_address") - override val disableFiledistributor = optionalYinstVar[java.lang.Boolean]("disable_filedistributor") -} - -object CloudConfigYinstVariables { - private class YinstVariable(yinstPkg:String, name: String) { - val value = Environment.optionalYinstVariable(yinstPkg + "." + name) - - def withDefault[T](defaultValue: T)(implicit c: Converter[T]) : T = { - value map { implicitly[Converter[T]].convert } getOrElse defaultValue - } - } - - private def yinstVar(setting:String, yinstPkg: String = "cloudconfig_server") = new YinstVariable(yinstPkg, setting) - - private def optionalYinstVar[T](setting:String, yinstPkg: String = "cloudconfig_server")(implicit c: Converter[T]): Optional[T] = { - Environment.optionalYinstVariable(yinstPkg + "." + setting) map ( c.convert ) - } - - implicit val configServerConverter: Converter[Array[ConfigServer]] = new Converter[Array[ConfigServer]] { - override def convert(s: String) = { - s split "[, ]" filter { !_.isEmpty } map { toConfigServer } - } - } - - implicit val stringArrayConverter: Converter[Array[String]] = new Converter[Array[String]] { - override def convert(s: String) = { - s split "[, ]" filter { !_.isEmpty } - } - } - - private def toConfigServer(hostPort: String): ConfigServer = Try { - val (host, portStr) = splitFirst(hostPort, ':') - val port = portStr map { _.toInt } - new ConfigServer(host, port) - }.getOrElse(throw new IllegalArgumentException(s"Invalid config server '$hostPort'")) - - private def splitFirst(string: String, separator: Character): (String, Option[String]) = { - val (beginning, endWithSeparator) = string span { _ != separator } - (beginning, tailOption(endWithSeparator)) - } - - def tailOption(s: String) = { - if (s.isEmpty) None - else Some(s.tail) - } - - implicit def toJavaOptional[U <% V, V](option: Option[U]): Optional[V] = option match { - case Some(u) => Optional.of(u: V) - case None => Optional.empty() - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala index 4764f4698f0..2aab88d8319 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala @@ -6,13 +6,13 @@ package com.yahoo.container.standalone * TODO: copied from standalone-container. Move to separate lib module instead. */ object Environment { - def optionalYinstVariable(name: String) = { + def optionalInstallVariable(name: String) = { env(name.replace(".", "__")). orElse(systemProperty(name)) //for unit testing } - def yinstVariable(name: String) = { - optionalYinstVariable(name). + def installVariable(name: String) = { + optionalInstallVariable(name). getOrElse { throw new IllegalStateException("Environment variable not set: " + name) } diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index 1f3ff652224..3ebea07bfe0 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -40,7 +40,7 @@ class StandaloneContainerApplication @Inject()(injector: Injector) extends Appli ConfiguredApplication.ensureVespaLoggingInitialized() - val applicationPath: Path = injectedApplicationPath.getOrElse(yinstApplicationPath) + val applicationPath: Path = injectedApplicationPath.getOrElse(installApplicationPath) val distributedFiles = new LocalFileDb(applicationPath) @@ -73,7 +73,7 @@ class StandaloneContainerApplication @Inject()(injector: Injector) extends Appli injector.getInstance(Key.get(classOf[Path], applicationPathName)) }.toOption - def yinstApplicationPath = path(yinstVariable(applicationLocationYinstVariable)) + def installApplicationPath = path(installVariable(applicationLocationInstallVariable)) override def start() { try { @@ -97,10 +97,10 @@ class StandaloneContainerApplication @Inject()(injector: Injector) extends Appli object StandaloneContainerApplication { val packageName = "standalone_jdisc_container" - val applicationLocationYinstVariable = s"$packageName.app_location" - val deploymentProfileYinstVariable = s"$packageName.deployment_profile" + val applicationLocationInstallVariable = s"$packageName.app_location" + val deploymentProfileInstallVariable = s"$packageName.deployment_profile" - val applicationPathName = Names.named(applicationLocationYinstVariable) + val applicationPathName = Names.named(applicationLocationInstallVariable) val disableNetworkingAnnotation = "JDisc.disableNetworking" val configModelRepoName = Names.named("ConfigModelRepo") @@ -143,9 +143,9 @@ object StandaloneContainerApplication { } def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { - optionalYinstVariable(deploymentProfileYinstVariable) match { + optionalInstallVariable(deploymentProfileInstallVariable) match { case None => new ContainerModelBuilder(true, networkingOption) - case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables) + case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables) case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") } } diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala index 585221a8795..efa3edb7b7e 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala @@ -11,8 +11,8 @@ import org.hamcrest.Matchers.{arrayContaining} * @author tonytv * @since 5. */ -class CloudConfigYinstVariablesTest { - def convert = CloudConfigYinstVariables.configServerConverter.convert _ +class CloudConfigInstallVariablesTest { + def convert = CloudConfigInstallVariables.configServerConverter.convert _ @Test def test_configserver_parsing { diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala index b70eefe0681..33f9a2e8594 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala @@ -23,7 +23,7 @@ object StandaloneContainer { def withStandaloneContainer[T](containerNode: Node) { withTempDirectory { applicationDirectory => - System.setProperty(StandaloneContainerApplication.applicationLocationYinstVariable, applicationDirectory.toString) + System.setProperty(StandaloneContainerApplication.applicationLocationInstallVariable, applicationDirectory.toString) createServicesXml(applicationDirectory, containerNode) val driver = TestDriver.newInjectedApplicationInstance(classOf[StandaloneContainerApplication]) diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp index cb76876c24e..2141dbf4b53 100644 --- a/storage/src/tests/bucketdb/initializertest.cpp +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -17,10 +17,13 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/vdstestlib/cppunit/dirconfig.hpp> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/log/log.h> LOG_SETUP(".test.bucketdb.initializing"); +using document::FixedBucketSpaces; + namespace storage { typedef uint16_t PartitionId; @@ -444,10 +447,12 @@ struct FakePersistenceLayer : public StorageLink { << " for which we should not get a request"; fatal(ost.str()); } else { - for (DiskData::const_iterator it2 = it->second.begin(); - it2 != it->second.end(); ++it2) - { - reply->getBuckets().push_back(it2->first); + if (cmd.getBucket().getBucketSpace() == FixedBucketSpaces::default_space()) { + for (DiskData::const_iterator it2 = it->second.begin(); + it2 != it->second.end(); ++it2) + { + reply->getBuckets().push_back(it2->first); + } } } if (!fatalError.empty()) { diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index b79894aee9a..77b661c2606 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -13,18 +13,45 @@ #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/storage/distributor/simpleclusterinformation.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <sstream> using namespace storage::api; using namespace storage::lib; using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using document::BucketSpace; +using document::FixedBucketSpaces; namespace storage::distributor { +namespace { + +std::string +getStringList(std::string s, uint32_t count) +{ + std::ostringstream ost; + for (uint32_t i = 0; i < count; ++i) { + if (i > 0) { + ost << ","; + } + ost << s; + } + return ost.str(); +} + +std::string +getRequestBucketInfoStrings(uint32_t count) +{ + return getStringList("Request bucket info", count); +} + +} + class BucketDBUpdaterTest : public CppUnit::TestFixture, public DistributorTestUtil { @@ -87,6 +114,9 @@ class BucketDBUpdaterTest : public CppUnit::TestFixture, CPPUNIT_TEST(batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted); CPPUNIT_TEST_SUITE_END(); +public: + BucketDBUpdaterTest(); + protected: void testNormalUsage(); void testDistributorChange(); @@ -161,10 +191,24 @@ protected: return clusterInfo; } + static std::string getNodeList(std::vector<uint16_t> nodes, size_t count); + + std::string getNodeList(std::vector<uint16_t> nodes); + + std::vector<uint16_t> + expandNodeVec(const std::vector<uint16_t> &nodes); + + std::vector<document::BucketSpace> _bucketSpaces; + + size_t messageCount(size_t messagesPerBucketSpace) const { + return messagesPerBucketSpace * _bucketSpaces.size(); + } + public: using OutdatedNodesMap = dbtransition::OutdatedNodesMap; void setUp() override { createLinks(); + _bucketSpaces = getBucketSpaces(); }; void tearDown() override { @@ -173,17 +217,17 @@ public: std::shared_ptr<RequestBucketInfoReply> getFakeBucketReply( const lib::ClusterState& state, - RequestBucketInfoCommand& cmd, + const RequestBucketInfoCommand& cmd, int storageIndex, - int bucketCount, - int invalidBucketCount = 0) + uint32_t bucketCount, + uint32_t invalidBucketCount = 0) { RequestBucketInfoReply* sreply = new RequestBucketInfoReply(cmd); sreply->setAddress(storageAddress(storageIndex)); api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); - for (int i=0; i<bucketCount + invalidBucketCount; i++) { + for (uint32_t i=0; i<bucketCount + invalidBucketCount; i++) { if (!getBucketDBUpdater().getDistributorComponent() .ownsBucketInState(state, makeDocumentBucket(document::BucketId(16, i)))) { continue; @@ -214,17 +258,17 @@ public: return std::shared_ptr<api::RequestBucketInfoReply>(sreply); } - void fakeBucketReply( - const lib::ClusterState& state, - RequestBucketInfoCommand& cmd, - int storageIndex, - int bucketCount, - int invalidBucketCount = 0) + void fakeBucketReply(const lib::ClusterState &state, + const api::StorageCommand &cmd, + uint32_t bucketCount, + uint32_t invalidBucketCount = 0) { + CPPUNIT_ASSERT(cmd.getType() == MessageType::REQUESTBUCKETINFO); + const api::StorageMessageAddress &address(*cmd.getAddress()); getBucketDBUpdater().onRequestBucketInfoReply( getFakeBucketReply(state, - cmd, - storageIndex, + dynamic_cast<const RequestBucketInfoCommand &>(cmd), + address.getIndex(), bucketCount, invalidBucketCount)); } @@ -332,22 +376,15 @@ public: } void completeBucketInfoGathering(const lib::ClusterState& state, - uint32_t expectedMsgs, - uint32_t nBuckets = 1) + size_t expectedMsgs, + uint32_t bucketCount = 1, + uint32_t invalidBucketCount = 0) { - CPPUNIT_ASSERT_EQUAL(size_t(expectedMsgs), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(expectedMsgs, _sender.commands.size()); for (uint32_t i = 0; i < _sender.commands.size(); i++) { - CPPUNIT_ASSERT(_sender.commands[i]->getType() == - MessageType::REQUESTBUCKETINFO); - - const api::StorageMessageAddress& address( - *_sender.commands[i]->getAddress()); - fakeBucketReply( - state, - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - address.getIndex(), - nBuckets); + fakeBucketReply(state, *_sender.commands[i], + bucketCount, invalidBucketCount); } } @@ -383,12 +420,12 @@ public: setSystemState(newState); - for (uint32_t i=0; i<numStorageNodes; i++) { + for (uint32_t i=0; i< messageCount(numStorageNodes); i++) { CPPUNIT_ASSERT(_sender.commands[i]->getType() == MessageType::REQUESTBUCKETINFO); const api::StorageMessageAddress *address = _sender.commands[i]->getAddress(); - CPPUNIT_ASSERT_EQUAL(i, (uint32_t)address->getIndex()); + CPPUNIT_ASSERT_EQUAL((uint32_t)(i / _bucketSpaces.size()), (uint32_t)address->getIndex()); } } @@ -401,11 +438,8 @@ public: "distributor:1 storage:%d", numStorageNodes)); lib::ClusterState newState(state); - for (uint32_t i=0; i<numStorageNodes; i++) { - fakeBucketReply(newState, - *((RequestBucketInfoCommand*)_sender.commands[i].get()), - i, - numBuckets); + for (uint32_t i=0; i< messageCount(numStorageNodes); i++) { + fakeBucketReply(newState, *_sender.commands[i], numBuckets); } assertCorrectBuckets(numBuckets, state); } @@ -580,12 +614,19 @@ public: CPPUNIT_TEST_SUITE_REGISTRATION(BucketDBUpdaterTest); +BucketDBUpdaterTest::BucketDBUpdaterTest() + : CppUnit::TestFixture(), + DistributorTestUtil(), + _bucketSpaces() +{ +} + void BucketDBUpdaterTest::testNormalUsage() { setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); // Ensure distribution hash is set correctly CPPUNIT_ASSERT_EQUAL( @@ -594,10 +635,8 @@ BucketDBUpdaterTest::testNormalUsage() dynamic_cast<const RequestBucketInfoCommand&>( *_sender.commands[0]).getDistributionHash()); - fakeBucketReply( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), - 0, 10); + fakeBucketReply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + *_sender.commands[0], 10); _sender.clear(); @@ -605,17 +644,12 @@ BucketDBUpdaterTest::testNormalUsage() // change is only implemented after completion of previous cluster state setSystemState(lib::ClusterState("distributor:2 .0.s:i storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); // Expect reply of first set SystemState request. CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.replies.size()); - for (uint32_t i = 0; i < 3; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, 10); - } - + completeBucketInfoGathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + messageCount(3), 10); assertCorrectBuckets(10, "distributor:2 storage:3"); } @@ -626,13 +660,9 @@ BucketDBUpdaterTest::testDistributorChange() // First sends request setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); - for (uint32_t i = 0; i < 3; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); - } + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); + completeBucketInfoGathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + messageCount(3), numBuckets); _sender.clear(); // No change from initializing to up (when done with last job) @@ -648,13 +678,9 @@ BucketDBUpdaterTest::testDistributorChange() // Removing distributor. Need to refetch new data from all nodes. setSystemState(lib::ClusterState("distributor:2 storage:3")); - CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); - for (uint32_t i = 0; i < 3; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:2 storage:3"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); - } + CPPUNIT_ASSERT_EQUAL(messageCount(3), _sender.commands.size()); + completeBucketInfoGathering(lib::ClusterState("distributor:2 storage:3"), + messageCount(3), numBuckets); _sender.clear(); assertCorrectBuckets(numBuckets, "distributor:2 storage:3"); } @@ -667,13 +693,9 @@ BucketDBUpdaterTest::testDistributorChangeWithGrouping() int numBuckets = 100; setSystemState(lib::ClusterState("distributor:6 storage:6")); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); - } + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); + completeBucketInfoGathering(lib::ClusterState("distributor:6 storage:6"), + messageCount(6), numBuckets); _sender.clear(); // Distributor going down in other group, no change @@ -693,7 +715,7 @@ BucketDBUpdaterTest::testDistributorChangeWithGrouping() // Changed grouping cause change setDistribution(getDistConfig6Nodes4Groups()); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); } void @@ -701,16 +723,13 @@ BucketDBUpdaterTest::testNormalUsageInitializing() { setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1 .0.s:i")); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); // Not yet passing on system state. CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 10, - 10); + completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), + _bucketSpaces.size(), 10, 10); assertCorrectBuckets(10, "distributor:1 storage:1"); @@ -727,12 +746,10 @@ BucketDBUpdaterTest::testNormalUsageInitializing() setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); // Send a new request bucket info up. - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); - fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 20); + completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), + _bucketSpaces.size(), 20); // Pass on cluster state and recheck buckets now. CPPUNIT_ASSERT_EQUAL(size_t(1), _senderDown.commands.size()); @@ -746,33 +763,34 @@ BucketDBUpdaterTest::testFailedRequestBucketInfo() setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate. - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); { - std::shared_ptr<api::RequestBucketInfoReply> reply = - getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 10); + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { + std::shared_ptr<api::RequestBucketInfoReply> reply = + getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *((RequestBucketInfoCommand*)_sender.commands[i].get()), + 0, + 10); + reply->setResult(api::ReturnCode::NOT_CONNECTED); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + } - reply->setResult(api::ReturnCode::NOT_CONNECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - // Trigger that delayed message is sent + // Trigger that delayed message is sent getClock().addSecondsToTime(10); getBucketDBUpdater().resendDelayedMessages(); } // Should be resent. - CPPUNIT_ASSERT_EQUAL(std::string("Request bucket info," - "Request bucket info"), + CPPUNIT_ASSERT_EQUAL(getRequestBucketInfoStrings(messageCount(2)), _sender.getCommands()); CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[1].get()), - 0, - 10); + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { + fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *_sender.commands[_bucketSpaces.size() + i], 10); + } for (int i=0; i<10; i++) { CPPUNIT_ASSERT_EQUAL( @@ -791,14 +809,14 @@ BucketDBUpdaterTest::testEncodeErrorHandling() { setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); // Not yet passing on system state. CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - { + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { std::shared_ptr<api::RequestBucketInfoReply> reply = getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), + *((RequestBucketInfoCommand*)_sender.commands[i].get()), 0, 10); @@ -815,21 +833,15 @@ BucketDBUpdaterTest::testDownWhileInit() setStorageNodes(3); fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *((RequestBucketInfoCommand*)_sender.commands[0].get()), - 0, - 5); + *_sender.commands[0], 5); setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *((RequestBucketInfoCommand*)_sender.commands[2].get()), - 2, - 5); + *_sender.commands[2], 5); fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *((RequestBucketInfoCommand*)_sender.commands[1].get()), - 1, - 5); + *_sender.commands[1], 5); } bool @@ -844,6 +856,42 @@ BucketDBUpdaterTest::bucketExistsThatHasNode(int bucketCount, uint16_t node) con return false; } +std::string +BucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes, size_t count) +{ + std::ostringstream ost; + bool first = true; + for (const auto &node : nodes) { + for (uint32_t i = 0; i < count; ++i) { + if (!first) { + ost << ","; + } + ost << node; + first = false; + } + } + return ost.str(); +} + +std::string +BucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes) +{ + return getNodeList(std::move(nodes), _bucketSpaces.size()); +} + +std::vector<uint16_t> +BucketDBUpdaterTest::expandNodeVec(const std::vector<uint16_t> &nodes) +{ + std::vector<uint16_t> res; + size_t count = _bucketSpaces.size(); + for (const auto &node : nodes) { + for (uint32_t i = 0; i < count; ++i) { + res.push_back(node); + } + } + return res; +} + void BucketDBUpdaterTest::testNodeDown() { @@ -903,16 +951,13 @@ BucketDBUpdaterTest::testInitializingWhileRecheck() lib::ClusterState systemState("distributor:1 storage:2 .0.s:i .0.i:0.1"); setSystemState(systemState); - CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(2), _sender.commands.size()); CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); - for (int i=0; i<2; i++) { - fakeBucketReply(systemState, - *((RequestBucketInfoCommand*)_sender.commands[i].get()), - i, - 100); + for (uint32_t i = 0; i < messageCount(2); ++i) { + fakeBucketReply(systemState, *_sender.commands[i], 100); } // Now we can pass on system state. @@ -931,35 +976,36 @@ BucketDBUpdaterTest::testBitChange() { setSystemState(lib::ClusterState("bits:14 storage:1 distributor:2")); - CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); - - CPPUNIT_ASSERT(_sender.commands[0]->getType() == MessageType::REQUESTBUCKETINFO); - - RequestBucketInfoReply* sreply = - new RequestBucketInfoReply(*((RequestBucketInfoCommand*)_sender.commands[0].get())); - sreply->setAddress(storageAddress(0)); - api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); - + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); + + for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { + CPPUNIT_ASSERT(_sender.commands[bsi]->getType() == MessageType::REQUESTBUCKETINFO); + const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.commands[bsi]); + RequestBucketInfoReply* sreply = new RequestBucketInfoReply(req); + sreply->setAddress(storageAddress(0)); + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { + int cnt=0; + for (int i=0; cnt < 2; i++) { + lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution(); + std::vector<uint16_t> distributors; + if (distribution.getIdealDistributorNode( + lib::ClusterState("redundancy:1 bits:14 storage:1 distributor:2"), + document::BucketId(16, i)) + == 0) + { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); - int cnt=0; - for (int i=0; cnt < 2; i++) { - lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution(); - std::vector<uint16_t> distributors; - if (distribution.getIdealDistributorNode( - lib::ClusterState("redundancy:1 bits:14 storage:1 distributor:2"), - document::BucketId(16, i)) - == 0) - { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); - - bucketlist.push_back(document::BucketId(16, i)); - cnt++; + bucketlist.push_back(document::BucketId(16, i)); + cnt++; + } + } } - } - getBucketDBUpdater().onRequestBucketInfoReply(std::shared_ptr<RequestBucketInfoReply>(sreply)); + getBucketDBUpdater().onRequestBucketInfoReply(std::shared_ptr<RequestBucketInfoReply>(sreply)); + } } CPPUNIT_ASSERT_EQUAL( @@ -975,29 +1021,31 @@ BucketDBUpdaterTest::testBitChange() _sender.clear(); setSystemState(lib::ClusterState("bits:16 storage:1 distributor:2")); - CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); - - CPPUNIT_ASSERT(_sender.commands[0]->getType() == MessageType::REQUESTBUCKETINFO); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); + for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { + + CPPUNIT_ASSERT(_sender.commands[bsi]->getType() == MessageType::REQUESTBUCKETINFO); + const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.commands[bsi]); + RequestBucketInfoReply* sreply = new RequestBucketInfoReply(req); + sreply->setAddress(storageAddress(0)); + sreply->setResult(api::ReturnCode::OK); + if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + + for (uint32_t i = 0; i < 3; ++i) { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); + } - RequestBucketInfoReply* sreply = - new RequestBucketInfoReply( - *((RequestBucketInfoCommand*)_sender.commands[0].get())); - sreply->setAddress(storageAddress(0)); - sreply->setResult(api::ReturnCode::OK); - api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, 4), + api::BucketInfo(10,1,1))); + } - for (uint32_t i = 0; i < 3; ++i) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); + getBucketDBUpdater().onRequestBucketInfoReply( + std::shared_ptr<RequestBucketInfoReply>(sreply)); } - - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, 4), - api::BucketInfo(10,1,1))); - - getBucketDBUpdater().onRequestBucketInfoReply( - std::shared_ptr<RequestBucketInfoReply>(sreply)); } CPPUNIT_ASSERT_EQUAL( @@ -1256,7 +1304,7 @@ void BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() { setSystemState(lib::ClusterState("distributor:1 storage:1")); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); { api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false); @@ -1266,18 +1314,16 @@ BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() getBucketDBUpdater().onNotifyBucketChange(cmd); } - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); - fakeBucketReply( - lib::ClusterState("distributor:1 storage:1"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), - 0, 10); + completeBucketInfoGathering(lib::ClusterState("distributor:1 storage:1"), + _bucketSpaces.size(), 10); - CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size() + 1, _sender.commands.size()); { api::RequestBucketInfoCommand& rbi( - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[1])); + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[_bucketSpaces.size()])); CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), rbi.getBuckets()[0]); } @@ -1286,10 +1332,10 @@ BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() // Queue must be cleared once pending state is enabled. { lib::ClusterState state("distributor:1 storage:2"); - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(state, expectedMsgs, dummyBucketsToReturn); } - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); { api::RequestBucketInfoCommand& rbi( dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0])); @@ -1527,32 +1573,32 @@ void BucketDBUpdaterTest::testPendingClusterStateSendMessages() { CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("cluster:d", "distributor:1 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1"), + getNodeList({0, 1}), getSentNodes("cluster:d", "distributor:1 storage:3 .2.s:m")); CPPUNIT_ASSERT_EQUAL( - std::string("2"), + getNodeList({2}), getSentNodes("distributor:1 storage:2", "distributor:1 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("2,3,4,5"), + getNodeList({2, 3, 4, 5}), getSentNodes("distributor:1 storage:2", "distributor:1 storage:6")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("distributor:4 storage:3", "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2,3"), + getNodeList({0, 1, 2, 3}), getSentNodes("distributor:4 storage:3", "distributor:4 .2.s:d storage:4")); @@ -1567,17 +1613,17 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:4 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("2"), + getNodeList({2}), getSentNodes("distributor:3 storage:3 .2.s:i", "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:3 storage:3 .1.s:d", "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("1,2,4"), + getNodeList({1, 2, 4}), getSentNodes("distributor:3 storage:4 .1.s:d .2.s:i", "distributor:3 storage:5")); @@ -1602,7 +1648,7 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 .2.s:m storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("distributor:3 .2.s:m storage:3", "distributor:3 .2.s:d storage:3")); @@ -1612,21 +1658,21 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodesDistributionChanged("distributor:3 storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1"), + getNodeList({0, 1}), getSentNodes("distributor:10 storage:2", "distributor:10 .1.s:d storage:2")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:2 storage:2", "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:2 storage:2 .1.s:d", "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); @@ -1636,7 +1682,7 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 .2.s:i storage:2")); CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2"), + getNodeList({0, 1, 2}), getSentNodes("distributor:3 storage:3", "distributor:3 .2.s:s storage:3")); @@ -1646,7 +1692,7 @@ BucketDBUpdaterTest::testPendingClusterStateSendMessages() "distributor:3 .2.s:d storage:3")); CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:3 storage:3 .1.s:m", "distributor:3 storage:3")); @@ -1672,7 +1718,7 @@ BucketDBUpdaterTest::testPendingClusterStateReceive() clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap, api::Timestamp(1))); - CPPUNIT_ASSERT_EQUAL(3, (int)sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(3), sender.commands.size()); sortSentMessagesByIndex(sender); @@ -1713,7 +1759,7 @@ BucketDBUpdaterTest::testPendingClusterStateWithGroupDown() // Entire group 1 goes down. Must refetch from all nodes. CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2,3,4,5"), + getNodeList({0, 1, 2, 3, 4, 5}), getSentNodes("distributor:6 storage:6", "distributor:6 .2.s:d .3.s:d storage:6")); @@ -1733,7 +1779,7 @@ BucketDBUpdaterTest::testPendingClusterStateWithGroupDownAndNoHandover() // Group is down, but config says to not do anything about it. CPPUNIT_ASSERT_EQUAL( - std::string(""), + getNodeList({0, 1, 2, 3, 4, 5}, _bucketSpaces.size() - 1), getSentNodes("distributor:6 storage:6", "distributor:6 .2.s:d .3.s:d storage:6")); } @@ -1973,7 +2019,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInCurrentState() document::BucketId bucket(16, 3); lib::ClusterState stateBefore("distributor:1 storage:1"); { - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } _sender.clear(); @@ -1988,7 +2034,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInCurrentState() lib::ClusterState stateAfter("distributor:3 storage:3"); { - uint32_t expectedMsgs = 2, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = messageCount(2), dummyBucketsToReturn = 1; setAndEnableClusterState(stateAfter, expectedMsgs, dummyBucketsToReturn); } CPPUNIT_ASSERT(!getBucketDBUpdater().getDistributorComponent() @@ -2005,7 +2051,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInPendingState() document::BucketId bucket(16, 3); lib::ClusterState stateBefore("distributor:1 storage:1"); { - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } _sender.clear(); @@ -2042,38 +2088,35 @@ BucketDBUpdaterTest::testClusterStateAlwaysSendsFullFetchWhenDistributionChangeP { lib::ClusterState stateBefore("distributor:6 storage:6"); { - uint32_t expectedMsgs = 6, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } _sender.clear(); std::string distConfig(getDistConfig6Nodes3Groups()); setDistribution(distConfig); sortSentMessagesByIndex(_sender); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); // Suddenly, a wild cluster state change appears! Even though this state // does not in itself imply any bucket changes, it will still overwrite the // pending cluster state and thus its state of pending bucket info requests. setSystemState(lib::ClusterState("distributor:6 .2.t:12345 storage:6")); - CPPUNIT_ASSERT_EQUAL(size_t(12), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(12), _sender.commands.size()); - // Send replies for first 6 (outdated requests). + // Send replies for first messageCount(6) (outdated requests). int numBuckets = 10; - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); + for (uint32_t i = 0; i < messageCount(6); ++i) { + fakeBucketReply(lib::ClusterState("distributor:6 storage:6"), + *_sender.commands[i], numBuckets); } // No change from these. assertCorrectBuckets(1, "distributor:6 storage:6"); // Send for current pending. - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 .2.t:12345 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i + 6]), - i, numBuckets); + for (uint32_t i = 0; i < messageCount(6); ++i) { + fakeBucketReply(lib::ClusterState("distributor:6 .2.t:12345 storage:6"), + *_sender.commands[i + messageCount(6)], + numBuckets); } assertCorrectBuckets(numBuckets, "distributor:6 storage:6"); _sender.clear(); @@ -2086,7 +2129,7 @@ BucketDBUpdaterTest::testClusterStateAlwaysSendsFullFetchWhenDistributionChangeP void BucketDBUpdaterTest::testChangedDistributionConfigTriggersRecoveryMode() { - setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), 6, 20); + setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), 20); _sender.clear(); // First cluster state; implicit scan of all buckets which does not // use normal recovery mode ticking-path. @@ -2098,13 +2141,11 @@ BucketDBUpdaterTest::testChangedDistributionConfigTriggersRecoveryMode() // No replies received yet, still no recovery mode. CPPUNIT_ASSERT(!_distributor->isInRecoveryMode()); - CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(messageCount(6), _sender.commands.size()); uint32_t numBuckets = 10; - for (uint32_t i = 0; i < 6; ++i) { - fakeBucketReply( - lib::ClusterState("distributor:6 storage:6"), - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), - i, numBuckets); + for (uint32_t i = 0; i < messageCount(6); ++i) { + fakeBucketReply(lib::ClusterState("distributor:6 storage:6"), + *_sender.commands[i], numBuckets); } // Pending cluster state (i.e. distribution) has been enabled, which should @@ -2118,7 +2159,7 @@ BucketDBUpdaterTest::testNewlyAddedBucketsHaveCurrentTimeAsGcTimestamp() getClock().setAbsoluteTimeInSeconds(101234); lib::ClusterState stateBefore("distributor:1 storage:1"); { - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); } @@ -2134,7 +2175,7 @@ BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() { { lib::ClusterState stateBefore("distributor:1 storage:1 .0.s:i"); - uint32_t expectedMsgs = 1, dummyBucketsToReturn = 0; + uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 0; // This step is required to make the distributor ready for accepting // the below explicit database insertion towards node 0. setAndEnableClusterState(stateBefore, expectedMsgs, @@ -2144,7 +2185,7 @@ BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() getClock().setAbsoluteTimeInSeconds(1000); lib::ClusterState state("distributor:1 storage:1"); setSystemState(state); - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); // Before replying with the bucket info, simulate the arrival of a mutation // reply that alters the state of the bucket with information that will be @@ -2168,11 +2209,9 @@ BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() // happening before t=1000 but receiving a reply at t>1000 does not affect // correctness, as this should contain the same bucket info as that // contained in the full bucket reply and the DB update is thus idempotent. - fakeBucketReply( - state, - dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), - 0, - bucketsReturned); + for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { + fakeBucketReply(state, *_sender.commands[i], bucketsReturned); + } BucketDatabase::Entry e(getBucket(bucket)); CPPUNIT_ASSERT_EQUAL(uint32_t(1), e->getNodeCount()); @@ -2231,8 +2270,9 @@ void BucketDBUpdaterTest::preemptedDistrChangeCarriesNodeSetOverToNextStateFetch() { CPPUNIT_ASSERT_EQUAL( - (nodeVec{0, 1, 2, 3, 4, 5}), - getSentNodesWithPreemption("version:1 distributor:6 storage:6", 6, + expandNodeVec({0, 1, 2, 3, 4, 5}), + getSentNodesWithPreemption("version:1 distributor:6 storage:6", + messageCount(6), "version:2 distributor:6 .5.s:d storage:6", "version:3 distributor:6 storage:6")); } @@ -2241,9 +2281,10 @@ void BucketDBUpdaterTest::preemptedStorChangeCarriesNodeSetOverToNextStateFetch() { CPPUNIT_ASSERT_EQUAL( - (nodeVec{2, 3}), + expandNodeVec({2, 3}), getSentNodesWithPreemption( - "version:1 distributor:6 storage:6 .2.s:d", 5, + "version:1 distributor:6 storage:6 .2.s:d", + messageCount(5), "version:2 distributor:6 storage:6 .2.s:d .3.s:d", "version:3 distributor:6 storage:6")); } @@ -2252,9 +2293,10 @@ void BucketDBUpdaterTest::preemptedStorageNodeDownMustBeReFetched() { CPPUNIT_ASSERT_EQUAL( - (nodeVec{2}), + expandNodeVec({2}), getSentNodesWithPreemption( - "version:1 distributor:6 storage:6", 6, + "version:1 distributor:6 storage:6", + messageCount(6), "version:2 distributor:6 storage:6 .2.s:d", "version:3 distributor:6 storage:6")); } @@ -2265,7 +2307,8 @@ BucketDBUpdaterTest::doNotSendToPreemptedNodeNowInDownState() CPPUNIT_ASSERT_EQUAL( nodeVec{}, getSentNodesWithPreemption( - "version:1 distributor:6 storage:6 .2.s:d", 5, + "version:1 distributor:6 storage:6 .2.s:d", + messageCount(5), "version:2 distributor:6 storage:6", // Sends to 2. "version:3 distributor:6 storage:6 .2.s:d")); // 2 down again. } @@ -2276,9 +2319,10 @@ BucketDBUpdaterTest::doNotSendToPreemptedNodeNotPartOfNewState() // Even though 100 nodes are preempted, not all of these should be part // of the request afterwards when only 6 are part of the state. CPPUNIT_ASSERT_EQUAL( - (nodeVec{0, 1, 2, 3, 4, 5}), + expandNodeVec({0, 1, 2, 3, 4, 5}), getSentNodesWithPreemption( - "version:1 distributor:6 storage:100", 100, + "version:1 distributor:6 storage:100", + messageCount(100), "version:2 distributor:5 .4.s:d storage:100", "version:3 distributor:6 storage:6")); } @@ -2288,7 +2332,7 @@ BucketDBUpdaterTest::outdatedNodeSetClearedAfterSuccessfulStateCompletion() { lib::ClusterState stateBefore( "version:1 distributor:6 storage:6 .1.t:1234"); - uint32_t expectedMsgs = 6, dummyBucketsToReturn = 10; + uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 10; setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); _sender.clear(); // New cluster state that should not by itself trigger any new fetches, @@ -2327,7 +2371,7 @@ BucketDBUpdaterTest::changedDiskSetTriggersReFetch() { // Same number of online disks, but the set of disks has changed. CPPUNIT_ASSERT_EQUAL( - std::string("1"), + getNodeList({1}), getSentNodes("distributor:2 storage:2 .1.d:3 .1.d.2.s:d", "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); } @@ -2343,7 +2387,7 @@ BucketDBUpdaterTest::changedDiskSetTriggersReFetch() void BucketDBUpdaterTest::nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer() { - uint32_t expectedMsgs = 3, dummyBucketsToReturn = 1; + uint32_t expectedMsgs = messageCount(3), dummyBucketsToReturn = 1; setAndEnableClusterState(lib::ClusterState("distributor:3 storage:3"), expectedMsgs, dummyBucketsToReturn); _sender.clear(); @@ -2371,11 +2415,11 @@ BucketDBUpdaterTest::nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer() // Attempt to apply state with {0, 1} set. This will compare the new state // with the previous state, which still has node 2. - expectedMsgs = 2; + expectedMsgs = messageCount(2); setAndEnableClusterState(lib::ClusterState("distributor:2 storage:2"), expectedMsgs, dummyBucketsToReturn); - CPPUNIT_ASSERT_EQUAL((nodeVec{0, 1}), getSendSet()); + CPPUNIT_ASSERT_EQUAL(expandNodeVec({0, 1}), getSendSet()); } void @@ -2413,7 +2457,7 @@ BucketDBUpdaterTest::changed_distribution_config_implies_ownership_transfer() void BucketDBUpdaterTest::transition_time_tracked_for_single_state_change() { - completeStateTransitionInSeconds("distributor:2 storage:2", 5, 2); + completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2)); CPPUNIT_ASSERT_EQUAL(uint64_t(5000), lastTransitionTimeInMillis()); } @@ -2421,8 +2465,8 @@ BucketDBUpdaterTest::transition_time_tracked_for_single_state_change() void BucketDBUpdaterTest::transition_time_reset_across_non_preempting_state_changes() { - completeStateTransitionInSeconds("distributor:2 storage:2", 5, 2); - completeStateTransitionInSeconds("distributor:2 storage:3", 3, 1); + completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2)); + completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(1)); CPPUNIT_ASSERT_EQUAL(uint64_t(3000), lastTransitionTimeInMillis()); } @@ -2431,13 +2475,13 @@ void BucketDBUpdaterTest::transition_time_tracked_for_distribution_config_change() { lib::ClusterState state("distributor:2 storage:2"); - setAndEnableClusterState(state, 2, 1); + setAndEnableClusterState(state, messageCount(2), 1); _sender.clear(); std::string distConfig(getDistConfig3Nodes1Group()); setDistribution(distConfig); getClock().addSecondsToTime(4); - completeBucketInfoGathering(state, 2); + completeBucketInfoGathering(state, messageCount(2)); CPPUNIT_ASSERT_EQUAL(uint64_t(4000), lastTransitionTimeInMillis()); } @@ -2451,7 +2495,7 @@ BucketDBUpdaterTest::transition_time_tracked_across_preempted_transitions() // Pre-empted with new state here, which will push out the old pending // state and replace it with a new one. We should still count the time // used processing the old state. - completeStateTransitionInSeconds("distributor:2 storage:3", 3, 3); + completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(3)); CPPUNIT_ASSERT_EQUAL(uint64_t(8000), lastTransitionTimeInMillis()); } diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index 2617c843912..64df05acb8f 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -9,10 +9,11 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <tests/common/hostreporter/util.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/storage/distributor/bucket_spaces_stats_provider.h> -namespace storage { -namespace distributor { +namespace storage::distributor { +using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; using End = vespalib::JsonStream::End; using File = vespalib::File; using Object = vespalib::JsonStream::Object; @@ -24,17 +25,26 @@ class DistributorHostInfoReporterTest : public CppUnit::TestFixture CPPUNIT_TEST(hostInfoAllInfo); CPPUNIT_TEST(generateExampleJson); CPPUNIT_TEST(noReportGeneratedIfDisabled); + CPPUNIT_TEST(bucket_spaces_stats_are_reported); CPPUNIT_TEST_SUITE_END(); void hostInfoWithPutLatenciesOnly(); void hostInfoAllInfo(); - void verifyReportedNodeLatencies( - const vespalib::Slime& root, - uint16_t node, - int64_t latencySum, - int64_t count); + void verifyReportedNodeLatencies(const vespalib::Slime& root, + uint16_t nodeIndex, + int64_t latencySum, + int64_t count); + void verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName, + size_t bucketsTotal, + size_t bucketsPending); + void verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName); void generateExampleJson(); void noReportGeneratedIfDisabled(); + void bucket_spaces_stats_are_reported(); }; CPPUNIT_TEST_SUITE_REGISTRATION(DistributorHostInfoReporterTest); @@ -71,6 +81,14 @@ struct MockedMinReplicaProvider : MinReplicaProvider } }; +struct MockedBucketSpacesStatsProvider : public BucketSpacesStatsProvider { + PerNodeBucketSpacesStats stats; + + PerNodeBucketSpacesStats getBucketSpacesStats() const override { + return stats; + } +}; + const vespalib::slime::Inspector& getNode(const vespalib::Slime& root, uint16_t nodeIndex) { @@ -97,37 +115,80 @@ getLatenciesForNode(const vespalib::Slime& root, uint16_t nodeIndex) return getNode(root, nodeIndex)["ops-latency"]; } -} // anon ns +const vespalib::slime::Inspector& +getBucketSpaceStats(const vespalib::Slime& root, uint16_t nodeIndex, const vespalib::string& bucketSpaceName) +{ + const auto& bucketSpaces = getNode(root, nodeIndex)["bucket-spaces"]; + for (size_t i = 0; i < bucketSpaces.entries(); ++i) { + if (bucketSpaces[i]["name"].asString().make_stringref() == bucketSpaceName) { + return bucketSpaces[i]; + } + } + throw std::runtime_error("No bucket space found with name " + bucketSpaceName); +} + +} void -DistributorHostInfoReporterTest::verifyReportedNodeLatencies( - const vespalib::Slime& root, - uint16_t node, - int64_t latencySum, - int64_t count) +DistributorHostInfoReporterTest::verifyReportedNodeLatencies(const vespalib::Slime& root, + uint16_t nodeIndex, + int64_t latencySum, + int64_t count) { - auto& latencies = getLatenciesForNode(root, node); + auto& latencies = getLatenciesForNode(root, nodeIndex); CPPUNIT_ASSERT_EQUAL(latencySum, latencies["put"]["latency-ms-sum"].asLong()); CPPUNIT_ASSERT_EQUAL(count, latencies["put"]["count"].asLong()); } void -DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() +DistributorHostInfoReporterTest::verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName, + size_t bucketsTotal, + size_t bucketsPending) +{ + const auto &stats = getBucketSpaceStats(root, nodeIndex, bucketSpaceName); + const auto &buckets = stats["buckets"]; + CPPUNIT_ASSERT_EQUAL(bucketsTotal, static_cast<size_t>(buckets["total"].asLong())); + CPPUNIT_ASSERT_EQUAL(bucketsPending, static_cast<size_t>(buckets["pending"].asLong())); +} + +void +DistributorHostInfoReporterTest::verifyBucketSpaceStats(const vespalib::Slime& root, + uint16_t nodeIndex, + const vespalib::string& bucketSpaceName) { + const auto &stats = getBucketSpaceStats(root, nodeIndex, bucketSpaceName); + CPPUNIT_ASSERT(!stats["buckets"].valid()); +} + +struct Fixture { MockedLatencyStatisticsProvider latencyStatsProvider; MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); + MockedBucketSpacesStatsProvider bucketSpacesStatsProvider; + DistributorHostInfoReporter reporter; + Fixture() + : latencyStatsProvider(), + minReplicaProvider(), + bucketSpacesStatsProvider(), + reporter(latencyStatsProvider, minReplicaProvider, bucketSpacesStatsProvider) + {} + ~Fixture() {} +}; +void +DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() +{ + Fixture f; NodeStatsSnapshot snapshot; snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = snapshot; + f.latencyStatsProvider.returnedSnapshot = snapshot; vespalib::Slime root; - util::reporterToSlime(reporter, root); + util::reporterToSlime(f.reporter, root); verifyReportedNodeLatencies(root, 0, 10000, 3); verifyReportedNodeLatencies(root, 5, 25000, 7); } @@ -135,23 +196,19 @@ DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() void DistributorHostInfoReporterTest::hostInfoAllInfo() { - MockedLatencyStatisticsProvider latencyStatsProvider; - MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); - + Fixture f; NodeStatsSnapshot latencySnapshot; latencySnapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; latencySnapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = latencySnapshot; + f.latencyStatsProvider.returnedSnapshot = latencySnapshot; std::unordered_map<uint16_t, uint32_t> minReplica; minReplica[0] = 2; minReplica[5] = 9; - minReplicaProvider.minReplica = minReplica; + f.minReplicaProvider.minReplica = minReplica; vespalib::Slime root; - util::reporterToSlime(reporter, root); + util::reporterToSlime(f.reporter, root); verifyReportedNodeLatencies(root, 0, 10000, 3); verifyReportedNodeLatencies(root, 5, 25000, 7); @@ -162,26 +219,28 @@ DistributorHostInfoReporterTest::hostInfoAllInfo() void DistributorHostInfoReporterTest::generateExampleJson() { - MockedLatencyStatisticsProvider latencyStatsProvider; - MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); - + Fixture f; NodeStatsSnapshot snapshot; snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = snapshot; + f.latencyStatsProvider.returnedSnapshot = snapshot; std::unordered_map<uint16_t, uint32_t> minReplica; minReplica[0] = 2; minReplica[5] = 9; - minReplicaProvider.minReplica = minReplica; + f.minReplicaProvider.minReplica = minReplica; + + PerNodeBucketSpacesStats stats; + stats[0]["default"] = BucketSpaceStats(11, 3); + stats[0]["global"] = BucketSpaceStats(13, 5); + stats[5]["default"] = BucketSpaceStats(); + f.bucketSpacesStatsProvider.stats = stats; vespalib::asciistream json; vespalib::JsonStream stream(json, true); stream << Object(); - reporter.report(stream); + f.reporter.report(stream); stream << End(); stream.finalize(); @@ -204,23 +263,46 @@ DistributorHostInfoReporterTest::generateExampleJson() void DistributorHostInfoReporterTest::noReportGeneratedIfDisabled() { - MockedLatencyStatisticsProvider latencyStatsProvider; - MockedMinReplicaProvider minReplicaProvider; - DistributorHostInfoReporter reporter(latencyStatsProvider, - minReplicaProvider); - reporter.enableReporting(false); + Fixture f; + f.reporter.enableReporting(false); NodeStatsSnapshot snapshot; snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; - latencyStatsProvider.returnedSnapshot = snapshot; + f.latencyStatsProvider.returnedSnapshot = snapshot; vespalib::Slime root; - util::reporterToSlime(reporter, root); + util::reporterToSlime(f.reporter, root); CPPUNIT_ASSERT_EQUAL(size_t(0), root.get().children()); } -} // distributor -} // storage +void +DistributorHostInfoReporterTest::bucket_spaces_stats_are_reported() +{ + Fixture f; + PerNodeBucketSpacesStats stats; + stats[1]["default"] = BucketSpaceStats(11, 3); + stats[1]["global"] = BucketSpaceStats(13, 5); + stats[2]["default"] = BucketSpaceStats(17, 7); + stats[2]["global"] = BucketSpaceStats(); + stats[3]["default"] = BucketSpaceStats(19, 11); + f.bucketSpacesStatsProvider.stats = stats; + + vespalib::Slime root; + util::reporterToSlime(f.reporter, root); + verifyBucketSpaceStats(root, 1, "default", 11, 3); + verifyBucketSpaceStats(root, 1, "global", 13, 5); + verifyBucketSpaceStats(root, 2, "default", 17, 7); + verifyBucketSpaceStats(root, 2, "global"); + verifyBucketSpaceStats(root, 3, "default", 19, 11); + try { + verifyBucketSpaceStats(root, 3, "global"); + CPPUNIT_ASSERT(false); + } catch (const std::runtime_error &ex) { + CPPUNIT_ASSERT("No bucket space found with name global" == vespalib::string(ex.what())); + } +} + +} diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 1640af0f871..d585c4d0d32 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <tests/distributor/distributortestutil.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/config/config-stor-distributormanager.h> @@ -18,6 +19,7 @@ using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using document::FixedBucketSpaces; namespace storage { @@ -58,6 +60,9 @@ class Distributor_Test : public CppUnit::TestFixture, CPPUNIT_TEST(closing_aborts_priority_queued_client_requests); CPPUNIT_TEST_SUITE_END(); +public: + Distributor_Test(); + protected: void testOperationGeneration(); void testOperationsGeneratedAndStartedWithoutDuplicates(); @@ -89,9 +94,12 @@ protected: void internal_messages_are_started_in_fifo_order_batch(); void closing_aborts_priority_queued_client_requests(); + std::vector<document::BucketSpace> _bucketSpaces; + public: void setUp() override { createLinks(); + _bucketSpaces = getBucketSpaces(); }; void tearDown() override { @@ -197,6 +205,13 @@ private: CPPUNIT_TEST_SUITE_REGISTRATION(Distributor_Test); +Distributor_Test::Distributor_Test() + : CppUnit::TestFixture(), + DistributorTestUtil(), + _bucketSpaces() +{ +} + void Distributor_Test::testOperationGeneration() { @@ -752,19 +767,23 @@ void Distributor_Test::sendDownClusterStateCommand() { } void Distributor_Test::replyToSingleRequestBucketInfoCommandWith1Bucket() { - CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); - CPPUNIT_ASSERT_EQUAL(api::MessageType::REQUESTBUCKETINFO, - _sender.commands[0]->getType()); - auto& bucketReq(static_cast<api::RequestBucketInfoCommand&>( - *_sender.commands[0])); - auto bucketReply = bucketReq.makeReply(); - // Make sure we have a bucket to route our remove op to, or we'd get - // an immediate reply anyway. - dynamic_cast<api::RequestBucketInfoReply&>(*bucketReply) - .getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(1, 1), - api::BucketInfo(20, 10, 12, 50, 60, true, true))); - _distributor->handleMessage(std::move(bucketReply)); + CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); + for (uint32_t i = 0; i < _sender.commands.size(); ++i) { + CPPUNIT_ASSERT_EQUAL(api::MessageType::REQUESTBUCKETINFO, + _sender.commands[i]->getType()); + auto& bucketReq(static_cast<api::RequestBucketInfoCommand&> + (*_sender.commands[i])); + auto bucketReply = bucketReq.makeReply(); + if (bucketReq.getBucketSpace() == FixedBucketSpaces::default_space()) { + // Make sure we have a bucket to route our remove op to, or we'd get + // an immediate reply anyway. + dynamic_cast<api::RequestBucketInfoReply&>(*bucketReply) + .getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry(document::BucketId(1, 1), + api::BucketInfo(20, 10, 12, 50, 60, true, true))); + } + _distributor->handleMessage(std::move(bucketReply)); + } _sender.commands.clear(); } diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index fedd836513b..8aa9ffadebe 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -376,4 +376,14 @@ DistributorTestUtil::getDistribution() const { return getBucketSpaceRepo().get(makeBucketSpace()).getDistribution(); } +std::vector<document::BucketSpace> +DistributorTestUtil::getBucketSpaces() const +{ + std::vector<document::BucketSpace> res; + for (const auto &repo : getBucketSpaceRepo()) { + res.push_back(repo.first); + } + return res; +} + } diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h index 19da0483165..45aaf6b1dc7 100644 --- a/storage/src/tests/distributor/distributortestutil.h +++ b/storage/src/tests/distributor/distributortestutil.h @@ -166,6 +166,7 @@ public: BucketDatabase::Entry getBucket(const document::BucketId& bId) const; + std::vector<document::BucketSpace> getBucketSpaces() const; protected: vdstestlib::DirConfig _config; std::unique_ptr<TestDistributorApp> _node; diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index 945ccfa1484..7103a89229d 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -9,11 +9,13 @@ #include <vespa/storageapi/message/visitor.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <tests/distributor/distributortestutil.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using document::FixedBucketSpaces; namespace storage { namespace distributor { @@ -22,9 +24,14 @@ class IdealStateManagerTest : public CppUnit::TestFixture, public DistributorTestUtil { public: - IdealStateManagerTest() {} + IdealStateManagerTest() + : CppUnit::TestFixture(), + DistributorTestUtil(), + _bucketSpaces() + {} void setUp() override { createLinks(); + _bucketSpaces = getBucketSpaces(); }; void tearDown() override { @@ -54,6 +61,9 @@ public: CPPUNIT_TEST(testBlockIdealStateOpsOnFullRequestBucketInfo); CPPUNIT_TEST(testBlockCheckForAllOperationsToSpecificBucket); CPPUNIT_TEST_SUITE_END(); +private: + std::vector<document::BucketSpace> _bucketSpaces; + std::string makeBucketStatusString(const std::string &defaultSpaceBucketStatus); }; CPPUNIT_TEST_SUITE_REGISTRATION(IdealStateManagerTest); @@ -91,8 +101,7 @@ IdealStateManagerTest::testStatusPage() { std::ostringstream ost; getIdealStateManager().getBucketStatus(ost); - CPPUNIT_ASSERT_EQUAL(std::string("<h2>default - BucketSpace(0x0000000000000001)</h2>\n" - "BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true,ready=false)]<br>\n" + CPPUNIT_ASSERT_EQUAL(makeBucketStatusString("BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true,ready=false)]<br>\n" "<b>BucketId(0x4000000000000005):</b> <i> : split: [Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) is " "higher than the configured limit of (100, 1000000)]</i> [node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true," "active=true,ready=false)]<br>\n"), @@ -113,8 +122,7 @@ IdealStateManagerTest::testDisabledStateChecker() { std::ostringstream ost; getIdealStateManager().getBucketStatus(ost); - CPPUNIT_ASSERT_EQUAL(std::string( - "<h2>default - BucketSpace(0x0000000000000001)</h2>\n" + CPPUNIT_ASSERT_EQUAL(makeBucketStatusString( "BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true,ready=false)]<br>\n" "<b>BucketId(0x4000000000000005):</b> <i> : split: [Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) is " "higher than the configured limit of (100, 1000000)]</i> [node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true," @@ -261,6 +269,19 @@ IdealStateManagerTest::testBlockCheckForAllOperationsToSpecificBucket() } } +std::string +IdealStateManagerTest::makeBucketStatusString(const std::string &defaultSpaceBucketStatus) +{ + std::ostringstream ost; + for (const auto &bucketSpace : _bucketSpaces) { + ost << "<h2>" << FixedBucketSpaces::to_string(bucketSpace) << " - " << bucketSpace << "</h2>\n"; + if (bucketSpace == FixedBucketSpaces::default_space()) { + ost << defaultSpaceBucketStatus; + } + } + return ost.str(); +} + } // distributor } // storage diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 48c24c06ea1..394df6024fd 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -60,7 +60,7 @@ void SimpleMaintenanceScannerTest::setUp() { _priorityGenerator.reset(new MockMaintenancePriorityGenerator()); - _bucketSpaceRepo = std::make_unique<DistributorBucketSpaceRepo>(false); + _bucketSpaceRepo = std::make_unique<DistributorBucketSpaceRepo>(); _priorityDb.reset(new SimpleBucketPriorityDatabase()); _scanner.reset(new SimpleMaintenanceScanner(*_priorityDb, *_priorityGenerator, *_bucketSpaceRepo)); } diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index ac1a046bd17..7895d9e4cd0 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -140,6 +140,8 @@ ModifiedBucketCheckerTest::testDoNotCheckModifiedBucketsIfAlreadyPending() expectCommandsAndSendReplies(0, 0); // After replies received, tick should send new requests again. replyToAll(messages, 0); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); _handler->tick(); expectCommandsAndSendReplies(3, 3); } @@ -177,13 +179,20 @@ ModifiedBucketCheckerTest::testRecheckRequestsAreChunked() _handler->tick(); expectCommandsAndSendReplies(1, 4); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); + // New round of fetching _handler->tick(); expectCommandsAndSendReplies(1, 10); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); // And done! _handler->tick(); expectCommandsAndSendReplies(0, 0); + _handler->tick(); // global bucket space ==> nothing to do + expectCommandsAndSendReplies(0, 0); } void diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp index 304c6eddddc..774ddb81578 100644 --- a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp @@ -11,6 +11,7 @@ ContentBucketSpaceRepo::ContentBucketSpaceRepo() : _map() { _map.emplace(document::FixedBucketSpaces::default_space(), std::make_unique<ContentBucketSpace>(document::FixedBucketSpaces::default_space())); + _map.emplace(document::FixedBucketSpaces::global_space(), std::make_unique<ContentBucketSpace>(document::FixedBucketSpaces::global_space())); } ContentBucketSpace & @@ -21,10 +22,6 @@ ContentBucketSpaceRepo::get(BucketSpace bucketSpace) const return *itr->second; } -void ContentBucketSpaceRepo::enableGlobalBucketSpace() { - _map.emplace(document::FixedBucketSpaces::global_space(), std::make_unique<ContentBucketSpace>(document::FixedBucketSpaces::global_space())); -} - ContentBucketSpaceRepo::BucketSpaces ContentBucketSpaceRepo::getBucketSpaces() const { diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.h b/storage/src/vespa/storage/common/content_bucket_space_repo.h index f0803d76282..0d4ddb86bcf 100644 --- a/storage/src/vespa/storage/common/content_bucket_space_repo.h +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.h @@ -24,8 +24,6 @@ public: BucketSpaceMap::const_iterator begin() const { return _map.begin(); } BucketSpaceMap::const_iterator end() const { return _map.end(); } - void enableGlobalBucketSpace(); - BucketSpaces getBucketSpaces() const; size_t getBucketMemoryUsage() const; diff --git a/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h new file mode 100644 index 00000000000..aeba827a976 --- /dev/null +++ b/storage/src/vespa/storage/distributor/bucket_spaces_stats_provider.h @@ -0,0 +1,48 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <map> +#include <unordered_map> + +namespace storage::distributor { + +/** + * Statistics for a single bucket space on a content node. + */ +class BucketSpaceStats { +private: + bool _valid; + size_t _bucketsTotal; + size_t _bucketsPending; +public: + BucketSpaceStats(size_t bucketsTotal_, size_t bucketsPending_) + : _valid(true), + _bucketsTotal(bucketsTotal_), + _bucketsPending(bucketsPending_) + {} + BucketSpaceStats() + : _valid(false), + _bucketsTotal(0), + _bucketsPending(0) + {} + bool valid() const { return _valid; } + size_t bucketsTotal() const { return _bucketsTotal; } + size_t bucketsPending() const { return _bucketsPending; } +}; + +/** + * Interface that provides snapshots of bucket spaces statistics per content node. + */ +class BucketSpacesStatsProvider { +public: + // Mapping from bucket space name to statistics for that bucket space. + using BucketSpacesStats = std::map<vespalib::string, BucketSpaceStats>; + // Mapping from content node index to statistics for all bucket spaces on that node. + using PerNodeBucketSpacesStats = std::unordered_map<uint16_t, BucketSpacesStats>; + + virtual ~BucketSpacesStatsProvider() {} + virtual PerNodeBucketSpacesStats getBucketSpacesStats() const = 0; +}; + +} diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index a559ce2ad1a..32554d02397 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -65,7 +65,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, framework::StatusReporter("distributor", "Distributor"), _compReg(compReg), _component(compReg, "distributor"), - _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>(_component.enableMultipleBucketSpaces())), + _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>()), _metrics(new DistributorMetricSet(_component.getLoadTypes()->getMetricLoadTypes())), _operationOwner(*this, _component.getClock()), _maintenanceOperationOwner(*this, _component.getClock()), @@ -94,7 +94,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _metricLock(), _maintenanceStats(), _bucketDbStats(), - _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), *this), + _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), *this, *this), _ownershipSafeTimeCalc( std::make_unique<OwnershipTransferSafeTimePointCalculator>( std::chrono::seconds(0))) // Set by config later @@ -527,10 +527,8 @@ Distributor::propagateDefaultDistribution( std::shared_ptr<const lib::Distribution> distribution) { _bucketSpaceRepo->get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - if (_component.enableMultipleBucketSpaces()) { - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); - _bucketSpaceRepo->get(document::FixedBucketSpaces::global_space()).setDistribution(std::move(global_distr)); - } + auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + _bucketSpaceRepo->get(document::FixedBucketSpaces::global_space()).setDistribution(std::move(global_distr)); } void diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 39695b26415..1cf0a4f1866 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -2,27 +2,27 @@ #pragma once -#include "idealstatemanager.h" +#include "bucket_spaces_stats_provider.h" #include "bucketdbupdater.h" -#include "pendingmessagetracker.h" +#include "distributor_host_info_reporter.h" +#include "distributorinterface.h" #include "externaloperationhandler.h" +#include "idealstatemanager.h" #include "min_replica_provider.h" -#include "distributorinterface.h" - +#include "pendingmessagetracker.h" #include "statusreporterdelegate.h" -#include "distributor_host_info_reporter.h" -#include <vespa/storage/distributor/maintenance/maintenancescheduler.h> -#include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> +#include <vespa/config/config.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/messagesender.h> +#include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> +#include <vespa/storage/distributor/maintenance/maintenancescheduler.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> -#include <vespa/config/config.h> +#include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/vespalib/util/sync.h> -#include <unordered_map> #include <queue> +#include <unordered_map> namespace storage { @@ -43,7 +43,8 @@ class Distributor : public StorageLink, public StatusDelegator, public framework::StatusReporter, public framework::TickingThread, - public MinReplicaProvider + public MinReplicaProvider, + public BucketSpacesStatsProvider { public: Distributor(DistributorComponentRegister&, @@ -197,6 +198,11 @@ private: */ std::unordered_map<uint16_t, uint32_t> getMinReplica() const override; + PerNodeBucketSpacesStats getBucketSpacesStats() const override { + // TODO: implement + return BucketSpacesStatsProvider::PerNodeBucketSpacesStats(); + } + /** * Atomically publish internal metrics to external ideal state metrics. * Takes metric lock. diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index 117776cd242..cc1b1eb9a17 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -14,13 +14,11 @@ using document::BucketSpace; namespace storage { namespace distributor { -DistributorBucketSpaceRepo::DistributorBucketSpaceRepo(bool enableGlobalBucketSpace) +DistributorBucketSpaceRepo::DistributorBucketSpaceRepo() : _map() { add(document::FixedBucketSpaces::default_space(), std::make_unique<DistributorBucketSpace>()); - if (enableGlobalBucketSpace) { - add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>()); - } + add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>()); } DistributorBucketSpaceRepo::~DistributorBucketSpaceRepo() = default; diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h index 7d7db240ad9..e30438771b2 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h @@ -21,7 +21,7 @@ private: BucketSpaceMap _map; public: - explicit DistributorBucketSpaceRepo(bool enableGlobalBucketSpace); + DistributorBucketSpaceRepo(); ~DistributorBucketSpaceRepo(); DistributorBucketSpaceRepo(const DistributorBucketSpaceRepo&&) = delete; diff --git a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp index 8ac471ee0b7..8d2e45655d2 100644 --- a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp +++ b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "bucket_spaces_stats_provider.h" #include "distributor_host_info_reporter.h" #include "min_replica_provider.h" #include "pendingmessagetracker.h" @@ -12,15 +13,19 @@ using std::unordered_map; namespace storage { namespace distributor { +using BucketSpacesStats = BucketSpacesStatsProvider::BucketSpacesStats; +using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; using Object = vespalib::JsonStream::Object; using Array = vespalib::JsonStream::Array; using End = vespalib::JsonStream::End; DistributorHostInfoReporter::DistributorHostInfoReporter( LatencyStatisticsProvider& latencyProvider, - MinReplicaProvider& minReplicaProvider) + MinReplicaProvider& minReplicaProvider, + BucketSpacesStatsProvider& bucketSpacesStatsProvider) : _latencyProvider(latencyProvider), _minReplicaProvider(minReplicaProvider), + _bucketSpacesStatsProvider(bucketSpacesStatsProvider), _enabled(true) { } @@ -38,15 +43,35 @@ writeOperationStats(vespalib::JsonStream& stream, } void +writeBucketSpacesStats(vespalib::JsonStream& stream, + const BucketSpacesStats& stats) +{ + for (const auto& elem : stats) { + stream << Object() << "name" << elem.first; + if (elem.second.valid()) { + stream << "buckets" << Object() + << "total" << elem.second.bucketsTotal() + << "pending" << elem.second.bucketsPending() + << End(); + } + stream << End(); + } +} + +void outputStorageNodes(vespalib::JsonStream& output, const unordered_map<uint16_t, NodeStats>& nodeStats, - const unordered_map<uint16_t, uint32_t>& minReplica) + const unordered_map<uint16_t, uint32_t>& minReplica, + const PerNodeBucketSpacesStats& bucketSpacesStats) { set<uint16_t> nodes; - for (auto& element : nodeStats) { + for (const auto& element : nodeStats) { nodes.insert(element.first); } - for (auto& element : minReplica) { + for (const auto& element : minReplica) { + nodes.insert(element.first); + } + for (const auto& element : bucketSpacesStats) { nodes.insert(element.first); } @@ -69,6 +94,13 @@ outputStorageNodes(vespalib::JsonStream& output, output << "min-current-replication-factor" << minReplicaIt->second; } + + auto bucketSpacesStatsIt = bucketSpacesStats.find(node); + if (bucketSpacesStatsIt != bucketSpacesStats.end()) { + output << "bucket-spaces" << Array(); + writeBucketSpacesStats(output, bucketSpacesStatsIt->second); + output << End(); + } } output << End(); } @@ -83,15 +115,15 @@ DistributorHostInfoReporter::report(vespalib::JsonStream& output) return; } - NodeStatsSnapshot nodeStats = _latencyProvider.getLatencyStatistics(); - std::unordered_map<uint16_t, uint32_t> minReplica = - _minReplicaProvider.getMinReplica(); + auto nodeStats = _latencyProvider.getLatencyStatistics(); + auto minReplica = _minReplicaProvider.getMinReplica(); + auto bucketSpacesStats = _bucketSpacesStatsProvider.getBucketSpacesStats(); output << "distributor" << Object(); { output << "storage-nodes" << Array(); - outputStorageNodes(output, nodeStats.nodeToStats, minReplica); + outputStorageNodes(output, nodeStats.nodeToStats, minReplica, bucketSpacesStats); output << End(); } diff --git a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h index 3cb878fc75c..3e6a02120c2 100644 --- a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h +++ b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.h @@ -7,6 +7,7 @@ namespace storage { namespace distributor { +class BucketSpacesStatsProvider; class LatencyStatisticsProvider; class MinReplicaProvider; struct OperationStats; @@ -15,7 +16,8 @@ class DistributorHostInfoReporter : public HostReporter { public: DistributorHostInfoReporter(LatencyStatisticsProvider& latencyProvider, - MinReplicaProvider& minReplicaProvider); + MinReplicaProvider& minReplicaProvider, + BucketSpacesStatsProvider& bucketSpacesStatsProvider); DistributorHostInfoReporter(const DistributorHostInfoReporter&) = delete; DistributorHostInfoReporter& operator=( @@ -43,6 +45,7 @@ public: private: LatencyStatisticsProvider& _latencyProvider; MinReplicaProvider& _minReplicaProvider; + BucketSpacesStatsProvider& _bucketSpacesStatsProvider; std::atomic<bool> _enabled; }; diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index 29c69b59760..888f1e816a1 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -42,18 +42,9 @@ void ServiceLayerComponentRegisterImpl::setDistribution(lib::Distribution::SP distribution) { _bucketSpaceRepo.get(document::FixedBucketSpaces::default_space()).setDistribution(distribution); - if (enableMultipleBucketSpaces()) { - auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); - _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); - } + auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); + _bucketSpaceRepo.get(document::FixedBucketSpaces::global_space()).setDistribution(global_distr); StorageComponentRegisterImpl::setDistribution(distribution); } -void ServiceLayerComponentRegisterImpl::setEnableMultipleBucketSpaces(bool enabled) { - StorageComponentRegisterImpl::setEnableMultipleBucketSpaces(enabled); - if (enabled) { - _bucketSpaceRepo.enableGlobalBucketSpace(); - } -} - } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index fc07f394fff..deb3b2c0767 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -38,7 +38,6 @@ public: void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; void setDiskCount(uint16_t count); void setDistribution(lib::Distribution::SP distribution) override; - void setEnableMultipleBucketSpaces(bool enabled) override; }; } // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 6d1dc8d587b..30d6f5bd45c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -5,6 +5,7 @@ #include "storagereply.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/stringutil.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <sstream> #include <vespa/log/bufferedlogger.h> @@ -42,6 +43,14 @@ namespace { vespalib::Version version5_0beta(4, 3, 0); } + +static bool +suppressEncodeWarning(const api::StorageMessage *msg) +{ + const auto *req = dynamic_cast<const api::RequestBucketInfoCommand *>(msg); + return ((req != nullptr) && (req->getBucketSpace() != document::FixedBucketSpaces::default_space())); +} + static mbus::Blob encodeMessage(const ProtocolSerialization & serializer, const mbus::Routable & routable, @@ -110,10 +119,13 @@ StorageProtocol::encode(const vespalib::Version& version, } } catch (std::exception & e) { - LOGBP(warning, "Failed to encode %s storage protocol message %s: %s", - version.toString().c_str(), - message.getInternalMessage()->toString().c_str(), - e.what()); + if (!(version < version6_0 && + suppressEncodeWarning(message.getInternalMessage().get()))) { + LOGBP(warning, "Failed to encode %s storage protocol message %s: %s", + version.toString().c_str(), + message.getInternalMessage()->toString().c_str(), + e.what()); + } } return mbus::Blob(0); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java new file mode 100644 index 00000000000..c050ddce2c6 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -0,0 +1,51 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identity; + +import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentityCertificate; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; + +import javax.net.ssl.SSLContext; +import java.io.File; +import java.nio.file.Paths; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; + +/** + * @author mortent + */ +public class SiaIdentityProvider implements AthenzIdentityProvider { + + private final AthenzDomain domain; + private final AthenzService service; + private final String path; + + public SiaIdentityProvider(SiaProviderConfig siaProviderConfig) { + this.domain = new AthenzDomain(siaProviderConfig.athenzDomain()); + this.service = new AthenzService(domain, siaProviderConfig.athenzService()); + this.path = siaProviderConfig.keyPathPrefix(); + } + + @Override + public String getDomain() { + return domain.getName(); + } + + @Override + public String getService() { + return service.getName(); + } + + @Override + public SSLContext getIdentitySslContext() { + X509Certificate certificate = Crypto.loadX509Certificate(Paths.get(path, "certs", String.format("%s.%s.cert.pem", getDomain(),getService())).toFile()); + PrivateKey privateKey = Crypto.loadPrivateKey(Paths.get(path, "keys", String.format("%s.%s.key.pem", getDomain(),getService())).toFile()); + + return new AthenzSslContextBuilder() + .withIdentityCertificate(new AthenzIdentityCertificate(certificate, privateKey)) + .build(); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/package-info.java new file mode 100644 index 00000000000..da31e72a1fa --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mortent + */ +@ExportPackage +package com.yahoo.vespa.athenz.identity; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def b/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def new file mode 100644 index 00000000000..f668ef544f7 --- /dev/null +++ b/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def @@ -0,0 +1,6 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.athenz.identity + +athenzDomain string +athenzService string +keyPathPrefix string diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java index 16a541f939c..bff9d2186e6 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java @@ -319,12 +319,12 @@ class IOThread implements Runnable, AutoCloseable { successfullHandshakes.getAndIncrement(); } catch (ServerResponseException ser) { executeProblemsCounter.incrementAndGet(); - log.log(Level.INFO, "Handshake did not work out " + endpoint, Exceptions.toMessageString(ser)); + log.info("Handshake did not work out " + endpoint + ": " + Exceptions.toMessageString(ser)); drainFirstDocumentsInQueueIfOld(); return ThreadState.CONNECTED; } catch (Throwable throwable) { // This cover IOException as well executeProblemsCounter.incrementAndGet(); - log.log(Level.INFO, "Problem with Handshake " + endpoint, Exceptions.toMessageString(throwable)); + log.info("Problem with Handshake " + endpoint + ": " + Exceptions.toMessageString(throwable)); drainFirstDocumentsInQueueIfOld(); client.close(); return ThreadState.DISCONNECTED; @@ -340,7 +340,7 @@ class IOThread implements Runnable, AutoCloseable { return ThreadState.CONNECTED; } catch (Throwable e) { // Covers IOException as well - log.info("Problems while handing data over to gateway " + endpoint + ": " + Exceptions.toMessageString(e)); + log.info("Problems while handing data over to gateway " + endpoint + ": " + Exceptions.toMessageString(e)); client.close(); return ThreadState.DISCONNECTED; } diff --git a/vespabase/conf/default-env.txt.in b/vespabase/conf/default-env.txt.in index 38a4d0cded5..3551cec9945 100644 --- a/vespabase/conf/default-env.txt.in +++ b/vespabase/conf/default-env.txt.in @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. fallback VESPA_HOME @CMAKE_INSTALL_PREFIX@ -override VESPA_USER vespa -override cloudconfig_server__disable_filedistributor true +override VESPA_USER @VESPA_USER@ |