diff options
762 files changed, 15749 insertions, 8248 deletions
diff --git a/.travis.yml b/.travis.yml index afb660eddc6..c1f970abad4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ branches: - master before_cache: - - sudo rm -rf $HOME/.m2/repository/com/yahoo + - sudo rm -rf $HOME/.m2/repository/com/yahoo/vespa - sudo rm -rf $HOME/.m2/repository/repository.xml - du --summarize --human-readable $HOME/.m2/repository - du --summarize --human-readable $HOME/.ccache diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 26e24be526c..c87589d7be2 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -33,6 +33,19 @@ <groupId>org.glassfish.jersey.media</groupId> <artifactId>jersey-media-json-jackson</artifactId> </exclusion> + <!--Exclude all Jackson bundles provided by JDisc --> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -45,13 +58,21 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </exclusion> + <!--Exclude all Jackson bundles provided by JDisc --> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> </exclusions> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jsr310</artifactId> - <scope>compile</scope> - </dependency> <!-- PROVIDED --> <dependency> @@ -102,6 +123,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.4.1</version> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java deleted file mode 100644 index 26a88896fb9..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.model.api.SuperModelProvider; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.jdisc.http.SecretStore; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.AthenzCertificateClient; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.CertificateClient; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.IdentityDocumentGenerator; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.IdentityDocumentServlet; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceConfirmationServlet; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceValidator; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.KeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.SecretStoreKeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.StatusServlet; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import org.eclipse.jetty.util.ssl.SslContextFactory; - -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import java.time.Duration; -import java.time.temporal.TemporalAmount; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * A component acting as both SIA for configserver and provides a lightweight Jetty instance hosting the InstanceConfirmation API - * - * @author bjorncs - */ -public class AthenzInstanceProviderService extends AbstractComponent { - - private static final Logger log = Logger.getLogger(AthenzInstanceProviderService.class.getName()); - - private final ScheduledExecutorService scheduler; - private final Server jetty; - - @Inject - public AthenzInstanceProviderService(AthenzProviderServiceConfig config, SuperModelProvider superModelProvider, - NodeRepository nodeRepository, Zone zone, SecretStore secretStore) { - this(config, new SecretStoreKeyProvider(secretStore, getZoneConfig(config, zone).secretName()), Executors.newSingleThreadScheduledExecutor(), - superModelProvider, nodeRepository, zone, new AthenzCertificateClient(config, getZoneConfig(config, zone)), createSslContextFactory()); - } - - private AthenzInstanceProviderService(AthenzProviderServiceConfig config, - KeyProvider keyProvider, - ScheduledExecutorService scheduler, - SuperModelProvider superModelProvider, - NodeRepository nodeRepository, - Zone zone, - CertificateClient certificateClient, - SslContextFactory sslContextFactory) { - this(config, scheduler, zone, sslContextFactory, - new InstanceValidator(keyProvider, superModelProvider), - new IdentityDocumentGenerator(config, getZoneConfig(config, zone), nodeRepository, zone, keyProvider), - new AthenzCertificateUpdater( - certificateClient, sslContextFactory, keyProvider, config, getZoneConfig(config, zone))); - } - - AthenzInstanceProviderService(AthenzProviderServiceConfig config, - ScheduledExecutorService scheduler, - Zone zone, - SslContextFactory sslContextFactory, - InstanceValidator instanceValidator, - IdentityDocumentGenerator identityDocumentGenerator, - AthenzCertificateUpdater reloader) { - // TODO: Enable for all systems. Currently enabled for CD system only - if (SystemName.cd.equals(zone.system())) { - this.scheduler = scheduler; - this.jetty = createJettyServer(config, sslContextFactory, instanceValidator, identityDocumentGenerator); - - // TODO Configurable update frequency - scheduler.scheduleAtFixedRate(reloader, 0, 1, TimeUnit.DAYS); - try { - jetty.start(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } else { - this.scheduler = null; - this.jetty = null; - } - } - - private static Server createJettyServer(AthenzProviderServiceConfig config, - SslContextFactory sslContextFactory, - InstanceValidator instanceValidator, - IdentityDocumentGenerator identityDocumentGenerator) { - Server server = new Server(); - ServerConnector connector = new ServerConnector(server, sslContextFactory); - connector.setPort(config.port()); - server.addConnector(connector); - - ServletHandler handler = new ServletHandler(); - InstanceConfirmationServlet instanceConfirmationServlet = new InstanceConfirmationServlet(instanceValidator); - handler.addServletWithMapping(new ServletHolder(instanceConfirmationServlet), config.apiPath() + "/instance"); - - IdentityDocumentServlet identityDocumentServlet = new IdentityDocumentServlet(identityDocumentGenerator); - handler.addServletWithMapping(new ServletHolder(identityDocumentServlet), config.apiPath() + "/identity-document"); - - handler.addServletWithMapping(StatusServlet.class, "/status.html"); - server.setHandler(handler); - return server; - - } - - private static AthenzProviderServiceConfig.Zones getZoneConfig(AthenzProviderServiceConfig config, Zone zone) { - String key = zone.environment().value() + "." + zone.region().value(); - return config.zones(key); - } - - static SslContextFactory createSslContextFactory() { - try { - SslContextFactory sslContextFactory = new SslContextFactory(); - sslContextFactory.setWantClientAuth(true); - sslContextFactory.setProtocol("TLS"); - sslContextFactory.setKeyManagerFactoryAlgorithm("SunX509"); - return sslContextFactory; - } catch (Exception e) { - throw new IllegalArgumentException("Failed to create SSL context factory: " + e.getMessage(), e); - } - } - - static class AthenzCertificateUpdater implements Runnable { - - // TODO Make expiry a configuration parameter - private static final TemporalAmount EXPIRY_TIME = Duration.ofDays(30); - private static final Logger log = Logger.getLogger(AthenzCertificateUpdater.class.getName()); - - private final CertificateClient certificateClient; - private final SslContextFactory sslContextFactory; - private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig config; - private final AthenzProviderServiceConfig.Zones zoneConfig; - - AthenzCertificateUpdater(CertificateClient certificateClient, - SslContextFactory sslContextFactory, - KeyProvider keyProvider, - AthenzProviderServiceConfig config, - AthenzProviderServiceConfig.Zones zoneConfig) { - this.certificateClient = certificateClient; - this.sslContextFactory = sslContextFactory; - this.keyProvider = keyProvider; - this.config = config; - this.zoneConfig = zoneConfig; - } - - @Override - public void run() { - try { - log.log(LogLevel.INFO, "Updating Athenz certificate through ZTS"); - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - X509Certificate certificate = certificateClient.updateCertificate(privateKey, EXPIRY_TIME); - - String dummyPassword = "athenz"; - KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(null); - keyStore.setKeyEntry("athenz", - privateKey, - dummyPassword.toCharArray(), - new Certificate[]{certificate}); - - sslContextFactory.reload(sslContextFactory -> { - sslContextFactory.setKeyStore(keyStore); - sslContextFactory.setKeyStorePassword(dummyPassword); - }); - log.log(LogLevel.INFO, "Athenz certificate reload successfully completed"); - } catch (Throwable e) { - log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + e.getMessage(), e); - } - } - } - - @Override - public void deconstruct() { - try { - // TODO: Fix deconstruct when setup properly in all zones - log.log(LogLevel.INFO, "Deconstructing Athenz provider service"); - if(scheduler != null) - scheduler.shutdown(); - if(jetty != null) - jetty.stop(); - if (scheduler != null && !scheduler.awaitTermination(1, TimeUnit.MINUTES)) { - log.log(LogLevel.ERROR, "Failed to stop certificate updater"); - } - } catch (InterruptedException e) { - log.log(LogLevel.ERROR, "Failed to stop certificate updater: " + e.getMessage(), e); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to stop Jetty: " + e.getMessage(), e); - } finally { - super.deconstruct(); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java new file mode 100644 index 00000000000..7910650ed5e --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java @@ -0,0 +1,117 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.http.ssl.SslKeyStoreConfigurator; +import com.yahoo.jdisc.http.ssl.SslKeyStoreContext; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.AthenzCertificateClient; + +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; + +/** + * @author bjorncs + */ +// TODO Cache certificate on disk +@SuppressWarnings("unused") // Component injected into Jetty connector factory +public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements SslKeyStoreConfigurator { + private static final Logger log = Logger.getLogger(AthenzSslKeyStoreConfigurator.class.getName()); + // TODO Make expiry and update frequency configurable parameters + private static final Duration CERTIFICATE_EXPIRY_TIME = Duration.ofDays(30); + private static final Duration CERTIFICATE_UPDATE_PERIOD = Duration.ofDays(7); + + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + private final AthenzCertificateClient certificateClient; + private final KeyProvider keyProvider; + private final AthenzProviderServiceConfig.Zones zoneConfig; + private final AtomicBoolean alreadyConfigured = new AtomicBoolean(); + private final Zone zone; + + @Inject + public AthenzSslKeyStoreConfigurator(KeyProvider keyProvider, + AthenzProviderServiceConfig config, + Zone zone) { + AthenzProviderServiceConfig.Zones zoneConfig = getZoneConfig(config, zone); + this.certificateClient = new AthenzCertificateClient(config, zoneConfig); + this.keyProvider = keyProvider; + this.zoneConfig = zoneConfig; + this.zone = zone; + } + + @Override + public void configure(SslKeyStoreContext sslKeyStoreContext) { + // TODO Remove this when main is ready + if (zone.system() != SystemName.cd) { + return; + } + if (alreadyConfigured.getAndSet(true)) { // For debugging purpose of SslKeyStoreConfigurator interface + throw new IllegalStateException("Already configured. configure() can only be called once."); + } + AthenzCertificateUpdater updater = new AthenzCertificateUpdater(sslKeyStoreContext); + scheduler.scheduleAtFixedRate(updater, /*initialDelay*/0, CERTIFICATE_UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); + } + + @Override + public void deconstruct() { + try { + scheduler.shutdownNow(); + scheduler.awaitTermination(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Failed to shutdown Athenz certificate updater on time", e); + } + } + + private class AthenzCertificateUpdater implements Runnable { + + private final SslKeyStoreContext sslKeyStoreContext; + + AthenzCertificateUpdater(SslKeyStoreContext sslKeyStoreContext) { + this.sslKeyStoreContext = sslKeyStoreContext; + } + + @Override + public void run() { + try { + log.log(LogLevel.INFO, "Updating Athenz certificate from ZTS"); + PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); + X509Certificate certificate = certificateClient.updateCertificate(privateKey, CERTIFICATE_EXPIRY_TIME); + verifyActualExperiy(certificate); + + String dummyPassword = "athenz"; + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null); + keyStore.setKeyEntry("athenz", privateKey, dummyPassword.toCharArray(), new Certificate[]{certificate}); + sslKeyStoreContext.updateKeyStore(keyStore, dummyPassword); + log.log(LogLevel.INFO, "Athenz certificate reload successfully completed"); + } catch (Throwable e) { + log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + e.getMessage(), e); + } + } + + private void verifyActualExperiy(X509Certificate certificate) { + Instant notAfter = certificate.getNotAfter().toInstant(); + Instant notBefore = certificate.getNotBefore().toInstant(); + if (!notBefore.plus(CERTIFICATE_EXPIRY_TIME).equals(notAfter)) { + Duration actualExpiry = Duration.between(notBefore, notAfter); + log.log(LogLevel.WARNING, + String.format("Expected expiry %s, got %s", CERTIFICATE_EXPIRY_TIME, actualExpiry)); + } + } + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/KeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java index 5a1d7e3c1ff..a72a2fcbc6c 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/KeyProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/KeyProvider.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import java.security.PrivateKey; import java.security.PublicKey; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSerializedPayload.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSerializedPayload.java new file mode 100644 index 00000000000..25733bf0075 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSerializedPayload.java @@ -0,0 +1,68 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.util.io.pem.PemObject; + +import java.io.IOException; +import java.io.StringWriter; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; + +/** + * Contains PEM formatted signed certificate + * + * @author freva + */ +public class CertificateSerializedPayload { + + @JsonProperty("certificate") @JsonSerialize(using = CertificateSerializer.class) + public final X509Certificate certificate; + + @JsonCreator + public CertificateSerializedPayload(@JsonProperty("certificate") X509Certificate certificate) { + this.certificate = certificate; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CertificateSerializedPayload that = (CertificateSerializedPayload) o; + + return certificate.equals(that.certificate); + } + + @Override + public int hashCode() { + return certificate.hashCode(); + } + + @Override + public String toString() { + return "CertificateSerializedPayload{" + + "certificate='" + certificate + '\'' + + '}'; + } + + public static class CertificateSerializer extends JsonSerializer<X509Certificate> { + @Override + public void serialize( + X509Certificate certificate, JsonGenerator gen, SerializerProvider serializers) throws IOException { + try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { + pemWriter.writeObject(new PemObject("CERTIFICATE", certificate.getEncoded())); + pemWriter.flush(); + gen.writeString(stringWriter.toString()); + } catch (CertificateEncodingException e) { + throw new RuntimeException("Failed to encode X509Certificate", e); + } + } + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java new file mode 100644 index 00000000000..2dc3f24664c --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSigner.java @@ -0,0 +1,150 @@ +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; +import com.yahoo.net.HostName; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.DERUTF8String; +import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers; +import org.bouncycastle.asn1.x500.AttributeTypeAndValue; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x500.style.BCStyle; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.Extensions; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest; + +import java.math.BigInteger; +import java.security.PrivateKey; +import java.security.Provider; +import java.security.PublicKey; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.Enumeration; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; + + +/** + * Signs Certificate Signing Reqest from tenant nodes. This certificate will be used + * by nodes to authenticate themselves when performing operations against the config + * server, such as updating node-repository or orchestrator. + * + * @author freva + */ +public class CertificateSigner { + + private static final Logger log = Logger.getLogger(CertificateSigner.class.getName()); + + static final String SIGNER_ALGORITHM = "SHA256withRSA"; + static final Duration CERTIFICATE_EXPIRATION = Duration.ofDays(30); + private static final List<ASN1ObjectIdentifier> ILLEGAL_EXTENSIONS = ImmutableList.of( + Extension.basicConstraints, Extension.subjectAlternativeName); + + private final JcaX509CertificateConverter certificateConverter = new JcaX509CertificateConverter(); + private final Provider provider = new BouncyCastleProvider(); + + private final PrivateKey caPrivateKey; + private final X500Name issuer; + private final Clock clock; + + @Inject + public CertificateSigner(KeyProvider keyProvider, AthenzProviderServiceConfig config, Zone zone) { + this(getPrivateKey(keyProvider, config, zone), HostName.getLocalhost(), Clock.systemUTC()); + } + + CertificateSigner(PrivateKey caPrivateKey, String configServerHostname, Clock clock) { + this.caPrivateKey = caPrivateKey; + this.issuer = new X500Name("CN=" + configServerHostname); + this.clock = clock; + } + + /** + * Signs the CSR if: + * <ul> + * <li>Common Name matches {@code remoteHostname}</li> + * <li>CSR does not contain any any of the extensions in {@code ILLEGAL_EXTENSIONS}</li> + * </ul> + */ + X509Certificate generateX509Certificate(PKCS10CertificationRequest certReq, String remoteHostname) { + verifyCertificateCommonName(certReq.getSubject(), remoteHostname); + verifyCertificateExtensions(certReq); + + Date notBefore = Date.from(clock.instant()); + Date notAfter = Date.from(clock.instant().plus(CERTIFICATE_EXPIRATION)); + + try { + PublicKey publicKey = new JcaPKCS10CertificationRequest(certReq).getPublicKey(); + X509v3CertificateBuilder caBuilder = new JcaX509v3CertificateBuilder( + issuer, BigInteger.valueOf(clock.millis()), notBefore, notAfter, certReq.getSubject(), publicKey) + + // Set Basic Constraints to false + .addExtension(Extension.basicConstraints, true, new BasicConstraints(false)); + + ContentSigner caSigner = new JcaContentSignerBuilder(SIGNER_ALGORITHM).build(caPrivateKey); + + return certificateConverter + .setProvider(provider) + .getCertificate(caBuilder.build(caSigner)); + } catch (Exception ex) { + log.log(LogLevel.ERROR, "Failed to generate X509 Certificate", ex); + throw new RuntimeException("Failed to generate X509 Certificate"); + } + } + + static void verifyCertificateCommonName(X500Name subject, String commonName) { + List<AttributeTypeAndValue> attributesAndValues = Arrays.stream(subject.getRDNs()) + .flatMap(rdn -> rdn.isMultiValued() ? + Stream.of(rdn.getTypesAndValues()) : Stream.of(rdn.getFirst())) + .filter(attr -> attr.getType() == BCStyle.CN) + .collect(Collectors.toList()); + + if (attributesAndValues.size() != 1) { + throw new IllegalArgumentException("Only 1 common name should be set"); + } + + String actualCommonName = DERUTF8String.getInstance(attributesAndValues.get(0).getValue()).getString(); + if (! actualCommonName.equals(commonName)) { + throw new IllegalArgumentException("Expected common name to be " + commonName + ", but was " + actualCommonName); + } + } + + @SuppressWarnings("unchecked") + static void verifyCertificateExtensions(PKCS10CertificationRequest request) { + List<String> illegalExt = Arrays + .stream(request.getAttributes(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) + .map(attribute -> Extensions.getInstance(attribute.getAttrValues().getObjectAt(0))) + .flatMap(ext -> Collections.list((Enumeration<ASN1ObjectIdentifier>) ext.oids()).stream()) + .filter(ILLEGAL_EXTENSIONS::contains) + .map(ASN1ObjectIdentifier::getId) + .collect(Collectors.toList()); + + if (! illegalExt.isEmpty()) { + throw new IllegalArgumentException("CSR contains illegal extensions: " + String.join(", ", illegalExt)); + } + } + + private static PrivateKey getPrivateKey(KeyProvider keyProvider, AthenzProviderServiceConfig config, Zone zone) { + AthenzProviderServiceConfig.Zones zoneConfig = getZoneConfig(config, zone); + return keyProvider.getPrivateKey(zoneConfig.secretVersion()); + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSignerResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSignerResource.java new file mode 100644 index 00000000000..417acf0e9b5 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSignerResource.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; + +import com.google.inject.Inject; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.log.LogLevel; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.security.cert.X509Certificate; +import java.util.logging.Logger; + +/** + * @author bjorncs + * @author freva + */ +@Path("/sign") +public class CertificateSignerResource { + + private static final Logger log = Logger.getLogger(CertificateSignerResource.class.getName()); + + private final CertificateSigner certificateSigner; + + @Inject + public CertificateSignerResource(@Component CertificateSigner certificateSigner) { + this.certificateSigner = certificateSigner; + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + public CertificateSerializedPayload generateCertificate(CsrSerializedPayload csrPayload, + @Context HttpServletRequest req) { + try { + String remoteHostname = req.getRemoteHost(); + PKCS10CertificationRequest csr = csrPayload.csr; + log.log(LogLevel.DEBUG, "Certification request from " + remoteHostname + ": " + csr); + X509Certificate certificate = certificateSigner.generateX509Certificate(csr, remoteHostname); + return new CertificateSerializedPayload(certificate); + } catch (RuntimeException e) { + log.log(LogLevel.ERROR, e.getMessage(), e); + throw new InternalServerErrorException(e.getMessage(), e); + } + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CsrSerializedPayload.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CsrSerializedPayload.java new file mode 100644 index 00000000000..f56214513aa --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CsrSerializedPayload.java @@ -0,0 +1,62 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; + +import java.io.IOException; +import java.io.StringReader; + +/** + * Contains PEM formatted Certificate Signing Request (CSR) + * + * @author freva + */ +public class CsrSerializedPayload { + + @JsonProperty("csr") public final PKCS10CertificationRequest csr; + + @JsonCreator + public CsrSerializedPayload(@JsonProperty("csr") @JsonDeserialize(using = CertificateRequestDeserializer.class) + PKCS10CertificationRequest csr) { + this.csr = csr; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CsrSerializedPayload that = (CsrSerializedPayload) o; + + return csr.equals(that.csr); + } + + @Override + public int hashCode() { + return csr.hashCode(); + } + + @Override + public String toString() { + return "CsrSerializedPayload{" + + "csr='" + csr + '\'' + + '}'; + } + + public static class CertificateRequestDeserializer extends JsonDeserializer<PKCS10CertificationRequest> { + @Override + public PKCS10CertificationRequest deserialize( + JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + try (PEMParser pemParser = new PEMParser(new StringReader(jsonParser.getValueAsString()))) { + return (PKCS10CertificationRequest) pemParser.readObject(); + } + } + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/IdentityDocument.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocument.java index 41ce5d969a7..bae8f6f03b6 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/IdentityDocument.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocument.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java index 55acf0b796c..4dd6881c07e 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; +import com.google.inject.Inject; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.IdentityDocument; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.ProviderUniqueId; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.SignedIdentityDocument; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -29,8 +29,12 @@ public class IdentityDocumentGenerator { private final String providerDomain; private final int signingSecretVersion; - public IdentityDocumentGenerator(AthenzProviderServiceConfig config, AthenzProviderServiceConfig.Zones zoneConfig, - NodeRepository nodeRepository, Zone zone, KeyProvider keyProvider) { + @Inject + public IdentityDocumentGenerator(AthenzProviderServiceConfig config, + NodeRepository nodeRepository, + Zone zone, + KeyProvider keyProvider) { + AthenzProviderServiceConfig.Zones zoneConfig = Utils.getZoneConfig(config, zone); this.nodeRepository = nodeRepository; this.zone = zone; this.keyProvider = keyProvider; @@ -41,7 +45,7 @@ public class IdentityDocumentGenerator { this.signingSecretVersion = zoneConfig.secretVersion(); } - public String generateSignedIdentityDocument(String hostname) { + public SignedIdentityDocument generateSignedIdentityDocument(String hostname) { Node node = nodeRepository.getNode(hostname).orElseThrow(() -> new RuntimeException("Unable to find node " + hostname)); try { IdentityDocument identityDocument = generateIdDocument(node); @@ -51,13 +55,12 @@ public class IdentityDocumentGenerator { Base64.getEncoder().encodeToString(identityDocumentString.getBytes()); Signature sigGenerator = Signature.getInstance("SHA512withRSA"); - // TODO: Get the correct version 0 ok for now PrivateKey privateKey = keyProvider.getPrivateKey(signingSecretVersion); sigGenerator.initSign(privateKey); sigGenerator.update(encodedIdentityDocument.getBytes()); String signature = Base64.getEncoder().encodeToString(sigGenerator.sign()); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + return new SignedIdentityDocument( encodedIdentityDocument, signature, SignedIdentityDocument.DEFAULT_KEY_VERSION, @@ -65,9 +68,7 @@ public class IdentityDocumentGenerator { toZoneDnsSuffix(zone, dnsSuffix), providerDomain + "." + providerService, ztsUrl, - SignedIdentityDocument.DEFAILT_DOCUMENT_VERSION - ); - return Utils.getMapper().writeValueAsString(signedIdentityDocument); + SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION); } catch (Exception e) { throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java new file mode 100644 index 00000000000..b3e5aee97b3 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; + +import com.google.inject.Inject; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.log.LogLevel; + +import javax.ws.rs.BadRequestException; +import javax.ws.rs.GET; +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +@Path("/identity-document") +public class IdentityDocumentResource { + + private static final Logger log = Logger.getLogger(IdentityDocumentResource.class.getName()); + + private final IdentityDocumentGenerator identityDocumentGenerator; + + @Inject + public IdentityDocumentResource(@Component IdentityDocumentGenerator identityDocumentGenerator) { + this.identityDocumentGenerator = identityDocumentGenerator; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + public SignedIdentityDocument getIdentityDocument(@QueryParam("hostname") String hostname) { + // TODO Use TLS client authentication instead of blindly trusting hostname + if (hostname == null) { + throw new BadRequestException("The 'hostname' query parameter is missing"); + } + try { + return identityDocumentGenerator.generateSignedIdentityDocument(hostname); + } catch (Exception e) { + String message = String.format("Unable to generate identity doument for '%s': %s", hostname, e.getMessage()); + log.log(LogLevel.ERROR, message, e); + throw new InternalServerErrorException(message, e); + } + } + +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/ProviderUniqueId.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/ProviderUniqueId.java index 810c75ef0c5..1de2292d2d0 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/ProviderUniqueId.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/ProviderUniqueId.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; import com.fasterxml.jackson.annotation.JsonProperty; @@ -76,4 +76,4 @@ public class ProviderUniqueId { public int hashCode() { return Objects.hash(tenant, application, environment, region, instance, clusterId, clusterIndex); } -}
\ No newline at end of file +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/SignedIdentityDocument.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/SignedIdentityDocument.java index 37f94d48a95..2545401f3ec 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/SignedIdentityDocument.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/SignedIdentityDocument.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -17,7 +17,7 @@ import java.util.Objects; public class SignedIdentityDocument { public static final int DEFAULT_KEY_VERSION = 0; - public static final int DEFAILT_DOCUMENT_VERSION = 1; + public static final int DEFAULT_DOCUMENT_VERSION = 1; @JsonProperty("identity-document")public final String rawIdentityDocument; @JsonIgnore public final IdentityDocument identityDocument; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java index dab1581f580..c6aee673f9c 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java @@ -17,7 +17,7 @@ import java.util.concurrent.TimeUnit; /** * @author bjorncs */ -public class AthenzCertificateClient implements CertificateClient { +public class AthenzCertificateClient { private final AthenzProviderServiceConfig config; private final AthenzPrincipalAuthority authority; @@ -29,7 +29,6 @@ public class AthenzCertificateClient implements CertificateClient { this.zoneConfig = zoneConfig; } - @Override public X509Certificate updateCertificate(PrivateKey privateKey, TemporalAmount expiryTime) { SimpleServiceIdentityProvider identityProvider = new SimpleServiceIdentityProvider( authority, zoneConfig.domain(), zoneConfig.serviceName(), diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CertificateClient.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CertificateClient.java deleted file mode 100644 index 6465873e092..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CertificateClient.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; - -import java.security.PrivateKey; -import java.security.cert.X509Certificate; -import java.time.temporal.TemporalAmount; - -/** - * @author bjorncs - */ -@FunctionalInterface -public interface CertificateClient { - X509Certificate updateCertificate(PrivateKey privateKey, TemporalAmount expiryTime); -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/FileBackedKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/FileBackedKeyProvider.java deleted file mode 100644 index 40a2a1dbcc9..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/FileBackedKeyProvider.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; - -import com.yahoo.athenz.auth.util.Crypto; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.security.PrivateKey; -import java.security.PublicKey; - -/** - * @author bjorncs - */ -public class FileBackedKeyProvider implements KeyProvider { - - private final String keyPathPrefix; - - public FileBackedKeyProvider(String keyPathPrefix) { - this.keyPathPrefix = keyPathPrefix; - } - - @Override - public PrivateKey getPrivateKey(int version) { - return Crypto.loadPrivateKey(readPemStringFromFile(new File(keyPathPrefix + ".priv." + version))); - } - - @Override - public PublicKey getPublicKey(int version) { - return Crypto.loadPublicKey(readPemStringFromFile(new File(keyPathPrefix + ".pub." + version))); - } - - private static String readPemStringFromFile(File file) { - try { - if (!file.exists() || !file.isFile()) { - throw new IllegalArgumentException("Key missing: " + file.getAbsolutePath()); - } - return new String(Files.readAllBytes(file.toPath())); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentServlet.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentServlet.java deleted file mode 100644 index a66fdf9d82f..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentServlet.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; - -import com.yahoo.log.LogLevel; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.logging.Logger; - -/** - * @author bjorncs - */ -public class IdentityDocumentServlet extends HttpServlet { - - private static final Logger log = Logger.getLogger(IdentityDocumentServlet.class.getName()); - - private final IdentityDocumentGenerator identityDocumentGenerator; - - public IdentityDocumentServlet(IdentityDocumentGenerator identityDocumentGenerator) { - this.identityDocumentGenerator = identityDocumentGenerator; - } - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - // TODO verify tls client cert - String hostname = req.getParameter("hostname"); - if (hostname == null) { - String message = "The 'hostname' parameter is missing"; - log.log(LogLevel.ERROR, message); - resp.sendError(HttpServletResponse.SC_BAD_REQUEST, message); - return; - } - try { - log.log(LogLevel.INFO, "Generating identity document for " + hostname); - String signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(hostname); - resp.setContentType("application/json"); - PrintWriter writer = resp.getWriter(); - writer.print(signedIdentityDocument); - writer.flush(); - } catch (Exception e) { - String message = String.format("Unable to generate identity doument [%s]", e.getMessage()); - log.log(LogLevel.ERROR, message); - resp.sendError(HttpServletResponse.SC_NOT_FOUND, message); - } - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceConfirmationServlet.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceConfirmationServlet.java deleted file mode 100644 index 766b95b443b..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceConfirmationServlet.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.InstanceConfirmation; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * A Servlet implementing the Athenz Service Provider InstanceConfirmation API - * - * @author bjorncs - */ -public class InstanceConfirmationServlet extends HttpServlet { - - private static final Logger log = Logger.getLogger(InstanceConfirmationServlet.class.getName()); - - private final InstanceValidator instanceValidator; - - public InstanceConfirmationServlet(InstanceValidator instanceValidator) { - this.instanceValidator = instanceValidator; - } - - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - // TODO Validate that request originates from ZTS - try { - String confirmationContent = toString(req.getReader()); - log.log(LogLevel.DEBUG, () -> "Confirmation content: " + confirmationContent); - InstanceConfirmation instanceConfirmation = - Utils.getMapper().readValue(confirmationContent, InstanceConfirmation.class); - log.log(LogLevel.DEBUG, () -> "Parsed confirmation content: " + instanceConfirmation.toString()); - if (!instanceValidator.isValidInstance(instanceConfirmation)) { - String message = "Invalid instance: " + instanceConfirmation; - log.log(LogLevel.ERROR, message); - resp.sendError(HttpServletResponse.SC_FORBIDDEN, message); - } else { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentType("application/json"); - resp.getWriter().write(Utils.getMapper().writeValueAsString(instanceConfirmation)); - } - } catch (JsonParseException | JsonMappingException e) { - String message = "InstanceConfirmation is not valid JSON"; - log.log(LogLevel.ERROR, message, e); - resp.sendError(HttpServletResponse.SC_BAD_REQUEST, message); - } - } - - private static String toString(Reader reader) throws IOException { - try (BufferedReader bufferedReader = new BufferedReader(reader)) { - return bufferedReader.lines().collect(Collectors.joining("\n")); - } - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java index 93abda1f9ea..e66131b6cf7 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java @@ -1,8 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +import com.google.inject.Inject; import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.http.SecretStore; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import java.security.KeyPair; import java.security.PrivateKey; @@ -10,19 +14,24 @@ import java.security.PublicKey; import java.util.HashMap; import java.util.Map; +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; + /** * @author mortent */ +@SuppressWarnings("unused") // Injected component public class SecretStoreKeyProvider implements KeyProvider { private final SecretStore secretStore; private final String secretName; private final Map<Integer, KeyPair> secrets; - - public SecretStoreKeyProvider(SecretStore secretStore, String secretName) { + @Inject + public SecretStoreKeyProvider(SecretStore secretStore, + Zone zone, + AthenzProviderServiceConfig config) { this.secretStore = secretStore; - this.secretName = secretName; + this.secretName = getZoneConfig(config, zone).secretName(); this.secrets = new HashMap<>(); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/StatusServlet.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/StatusServlet.java deleted file mode 100644 index fd5ba5843aa..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/StatusServlet.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; - -/** - * A simple status servlet that should return status code 200 as long as the provider service servlet is up. - * - * @author bjorncs - */ -public class StatusServlet extends HttpServlet { - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.setStatus(HttpServletResponse.SC_OK); - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java index d81ec183fd4..ad54aa341bf 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; /** * @author bjorncs @@ -20,4 +22,10 @@ public class Utils { mapper.registerModule(new JavaTimeModule()); return mapper; } + + public static AthenzProviderServiceConfig.Zones getZoneConfig(AthenzProviderServiceConfig config, Zone zone) { + String key = zone.environment().value() + "." + zone.region().value(); + return config.zones(key); + } + } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/InstanceConfirmation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmation.java index ade42968e58..7b2725a8d95 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/model/InstanceConfirmation.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmation.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument.SignedIdentityDocument; import java.io.IOException; import java.util.HashMap; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java new file mode 100644 index 00000000000..5c93bf423d3 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java @@ -0,0 +1,41 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; + +import com.google.inject.Inject; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.log.LogLevel; + +import javax.ws.rs.Consumes; +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +@Path("/instance") +public class InstanceConfirmationResource { + + private static final Logger log = Logger.getLogger(InstanceConfirmationResource.class.getName()); + + private final InstanceValidator instanceValidator; + + @Inject + public InstanceConfirmationResource(@Component InstanceValidator instanceValidator) { + this.instanceValidator = instanceValidator; + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public InstanceConfirmation confirmInstance(InstanceConfirmation instanceConfirmation) { + if (!instanceValidator.isValidInstance(instanceConfirmation)) { + log.log(LogLevel.ERROR, "Invalid instance: " + instanceConfirmation); + throw new ForbiddenException("Instance is invalid"); + } + return instanceConfirmation; + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 427f35c41d8..69c5d961b7e 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -1,14 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; +import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.ProviderUniqueId; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.SignedIdentityDocument; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument.ProviderUniqueId; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument.SignedIdentityDocument; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -33,6 +34,7 @@ public class InstanceValidator { private final KeyProvider keyProvider; private final SuperModelProvider superModelProvider; + @Inject public InstanceValidator(KeyProvider keyProvider, SuperModelProvider superModelProvider) { this.keyProvider = keyProvider; this.superModelProvider = superModelProvider; @@ -64,7 +66,7 @@ public class InstanceValidator { return isSignatureValid(publicKey, signedIdentityDocument.rawIdentityDocument, signedIdentityDocument.signature); } - static boolean isSignatureValid(PublicKey publicKey, String rawIdentityDocument, String signature) { + public static boolean isSignatureValid(PublicKey publicKey, String rawIdentityDocument, String signature) { try { Signature signatureVerifier = Signature.getInstance("SHA512withRSA"); signatureVerifier.initVerify(publicKey); diff --git a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def b/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def index 4aad9a4eae2..13cc78b0bd0 100644 --- a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def +++ b/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def @@ -13,12 +13,6 @@ zones{}.secretName string # Secret version zones{}.secretVersion int -# HTTPS port for Athenz Provider Service endpoint -port int default=8443 - -# InstanceConfirmation API path -apiPath string default="/athenz/v1/provider" - # Athenz principal authority header name athenzPrincipalHeaderName string default="Athenz-Principal-Auth" diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java deleted file mode 100644 index bf0746aee7e..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice; - -import athenz.shade.zts.jersey.repackaged.com.google.common.collect.ImmutableMap; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AthenzInstanceProviderService.AthenzCertificateUpdater; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.CertificateClient; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.IdentityDocumentGenerator; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceValidator; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.KeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.IdentityDocument; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.ProviderUniqueId; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.SignedIdentityDocument; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContextBuilder; -import org.bouncycastle.asn1.ASN1ObjectIdentifier; -import org.bouncycastle.asn1.x500.X500Name; -import org.bouncycastle.asn1.x509.BasicConstraints; -import org.bouncycastle.cert.CertIOException; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.operator.ContentSigner; -import org.bouncycastle.operator.OperatorCreationException; -import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.junit.Test; - -import javax.net.ssl.SSLContext; -import java.io.IOException; -import java.math.BigInteger; -import java.security.KeyManagementException; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.PublicKey; -import java.security.Signature; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.time.Instant; -import java.time.temporal.TemporalAmount; -import java.util.Base64; -import java.util.Calendar; -import java.util.Date; -import java.util.concurrent.ScheduledExecutorService; -import java.util.logging.Logger; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author bjorncs - */ -public class AthenzInstanceProviderServiceTest { - - private static final Logger log = Logger.getLogger(AthenzInstanceProviderServiceTest.class.getName()); - private static final int PORT = 12345; - private static final Zone ZONE = new Zone(SystemName.cd, Environment.dev, RegionName.from("us-north-1")); - - @Test - public void provider_service_hosts_endpoint_secured_with_tls() throws Exception { - String domain = "domain"; - String service = "service"; - - AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); - PrivateKey privateKey = keyProvider.getPrivateKey(0); - AthenzProviderServiceConfig config = getAthenzProviderConfig(domain, service, "vespa.dns.suffix", ZONE); - SslContextFactory sslContextFactory = AthenzInstanceProviderService.createSslContextFactory(); - AthenzCertificateUpdater certificateUpdater = new AthenzCertificateUpdater( - new SelfSignedCertificateClient(keyProvider.getKeyPair(), config, getZoneConfig(config, ZONE)), - sslContextFactory, - keyProvider, - config, - getZoneConfig(config, ZONE)); - - ScheduledExecutorService executor = mock(ScheduledExecutorService.class); - when(executor.awaitTermination(anyLong(), any())).thenReturn(true); - - InstanceValidator instanceValidator = mock(InstanceValidator.class); - when(instanceValidator.isValidInstance(any())).thenReturn(true); - - IdentityDocumentGenerator identityDocumentGenerator = mock(IdentityDocumentGenerator.class); - - AthenzInstanceProviderService athenzInstanceProviderService = new AthenzInstanceProviderService( - config, executor, ZONE, sslContextFactory, instanceValidator, identityDocumentGenerator, certificateUpdater); - - try (CloseableHttpClient client = createHttpClient(domain, service)) { - assertFalse(getStatus(client)); - certificateUpdater.run(); - assertTrue(getStatus(client)); - assertInstanceConfirmationSucceeds(client, privateKey); - certificateUpdater.run(); - assertTrue(getStatus(client)); - assertInstanceConfirmationSucceeds(client, privateKey); - } finally { - athenzInstanceProviderService.deconstruct(); - } - } - - public static AthenzProviderServiceConfig getAthenzProviderConfig(String domain, String service, String dnsSuffix, Zone zone) { - AthenzProviderServiceConfig.Zones.Builder zoneConfig = - new AthenzProviderServiceConfig.Zones.Builder() - .serviceName(service) - .secretVersion(0) - .domain(domain) - .secretName("s3cr3t"); - - return new AthenzProviderServiceConfig( - new AthenzProviderServiceConfig.Builder() - .zones(ImmutableMap.of(zone.environment().value() + "." + zone.region().value(), zoneConfig)) - .port(PORT) - .certDnsSuffix(dnsSuffix) - .ztsUrl("localhost/zts") - .athenzPrincipalHeaderName("Athenz-Principal-Auth") - .apiPath("")); - - } - - public static AthenzProviderServiceConfig.Zones getZoneConfig(AthenzProviderServiceConfig config, Zone zone) { - return config.zones(zone.environment().value() + "." + zone.region().value()); - } - - private static boolean getStatus(HttpClient client) { - try { - HttpResponse response = client.execute(new HttpGet("https://localhost:" + PORT + "/status.html")); - return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; - } catch (Exception e) { - log.log(LogLevel.INFO, "Status.html failed: " + e); - return false; - } - } - - private static void assertInstanceConfirmationSucceeds(HttpClient client, PrivateKey privateKey) throws IOException { - HttpPost httpPost = new HttpPost("https://localhost:" + PORT + "/instance"); - httpPost.setEntity(createInstanceConfirmation(privateKey)); - HttpResponse response = client.execute(httpPost); - assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); - } - - private static CloseableHttpClient createHttpClient(String domain, String service) - throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException { - SSLContext sslContext = new SSLContextBuilder() - .loadTrustMaterial(null, (certificateChain, ignoredAuthType) -> - certificateChain[0].getSubjectX500Principal().getName().equals("CN=" + domain + "." + service)) - .build(); - - return HttpClients.custom() - .setSslcontext(sslContext) - .setSSLHostnameVerifier(new NoopHostnameVerifier()) - .build(); - } - - private static HttpEntity createInstanceConfirmation(PrivateKey privateKey) { - IdentityDocument identityDocument = new IdentityDocument( - new ProviderUniqueId("tenant", "application", "environment", "region", "instance", "cluster-id", 0), - "hostname", - "instance-hostname", - Instant.now()); - try { - ObjectMapper mapper = Utils.getMapper(); - String encodedIdentityDocument = - Base64.getEncoder().encodeToString(mapper.writeValueAsString(identityDocument).getBytes()); - Signature sigGenerator = Signature.getInstance("SHA512withRSA"); - sigGenerator.initSign(privateKey); - sigGenerator.update(encodedIdentityDocument.getBytes()); - - InstanceConfirmation instanceConfirmation = new InstanceConfirmation( - "provider", "domain", "service", - new SignedIdentityDocument(encodedIdentityDocument, - Base64.getEncoder().encodeToString(sigGenerator.sign()), - 0, - identityDocument.providerUniqueId.asString(), - "dnssuffix", - "service", - "localhost/zts", - 1)); - return new StringEntity(mapper.writeValueAsString(instanceConfirmation)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static class AutoGeneratedKeyProvider implements KeyProvider { - - private final KeyPair keyPair; - - public AutoGeneratedKeyProvider() { - try { - KeyPairGenerator rsa = KeyPairGenerator.getInstance("RSA"); - rsa.initialize(2048); - keyPair = rsa.genKeyPair(); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - @Override - public PrivateKey getPrivateKey(int version) { - return keyPair.getPrivate(); - } - - @Override - public PublicKey getPublicKey(int version) { - return keyPair.getPublic(); - } - - public KeyPair getKeyPair() { - return keyPair; - } - } - - private static class SelfSignedCertificateClient implements CertificateClient { - - private final KeyPair keyPair; - private final AthenzProviderServiceConfig config; - private final AthenzProviderServiceConfig.Zones zoneConfig; - - private SelfSignedCertificateClient(KeyPair keyPair, AthenzProviderServiceConfig config, - AthenzProviderServiceConfig.Zones zoneConfig) { - this.keyPair = keyPair; - this.config = config; - this.zoneConfig = zoneConfig; - } - - @Override - public X509Certificate updateCertificate(PrivateKey privateKey, TemporalAmount expiryTime) { - try { - ContentSigner contentSigner = new JcaContentSignerBuilder("SHA512WithRSA").build(keyPair.getPrivate()); - X500Name dnName = new X500Name("CN=" + zoneConfig.domain() + "." + zoneConfig.serviceName()); - Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.HOUR, 1); - Date endDate = calendar.getTime(); - JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder( - dnName, BigInteger.ONE, new Date(), endDate, dnName, keyPair.getPublic()); - certBuilder.addExtension(new ASN1ObjectIdentifier("2.5.29.19"), true, new BasicConstraints(true)); - - return new JcaX509CertificateConverter() - .setProvider(new BouncyCastleProvider()) - .getCertificate(certBuilder.build(contentSigner)); - } catch (CertificateException | CertIOException | OperatorCreationException e) { - throw new RuntimeException(e); - } - } - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java new file mode 100644 index 00000000000..ca6b5529b08 --- /dev/null +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AutoGeneratedKeyProvider.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.PublicKey; + +/** + * @author bjorncs + */ +public class AutoGeneratedKeyProvider implements KeyProvider { + + private final KeyPair keyPair; + + public AutoGeneratedKeyProvider() { + try { + KeyPairGenerator rsa = KeyPairGenerator.getInstance("RSA"); + rsa.initialize(2048); + keyPair = rsa.genKeyPair(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + @Override + public PrivateKey getPrivateKey(int version) { + return keyPair.getPrivate(); + } + + @Override + public PublicKey getPublicKey(int version) { + return keyPair.getPublic(); + } + + public KeyPair getKeyPair() { + return keyPair; + } +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java new file mode 100644 index 00000000000..c09a9fb1740 --- /dev/null +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; + +/** + * @author bjorncs + */ +public class TestUtils { + + public static AthenzProviderServiceConfig getAthenzProviderConfig(String domain, + String service, + String dnsSuffix, + Zone zone) { + AthenzProviderServiceConfig.Zones.Builder zoneConfig = + new AthenzProviderServiceConfig.Zones.Builder() + .serviceName(service) + .secretVersion(0) + .domain(domain) + .secretName("s3cr3t"); + return new AthenzProviderServiceConfig( + new AthenzProviderServiceConfig.Builder() + .zones(ImmutableMap.of(zone.environment().value() + "." + zone.region().value(), zoneConfig)) + .certDnsSuffix(dnsSuffix) + .ztsUrl("localhost/zts") + .athenzPrincipalHeaderName("Athenz-Principal-Auth")); + } + +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSignerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSignerTest.java new file mode 100644 index 00000000000..e691da0b2c3 --- /dev/null +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CertificateSignerTest.java @@ -0,0 +1,134 @@ +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; + +import com.yahoo.test.ManualClock; +import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.Extensions; +import org.bouncycastle.asn1.x509.ExtensionsGenerator; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder; +import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder; +import org.junit.Test; + +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author freva + */ +public class CertificateSignerTest { + + private final KeyPair clientKeyPair = getKeyPair(); + + private final long startTime = 1234567890000L; + private final KeyPair caKeyPair = getKeyPair(); + private final String cfgServerHostname = "cfg1.us-north-1.vespa.domain.tld"; + private final ManualClock clock = new ManualClock(Instant.ofEpochMilli(startTime)); + private final CertificateSigner signer = new CertificateSigner(caKeyPair.getPrivate(), cfgServerHostname, clock); + + private final String requestersHostname = "tenant-123.us-north-1.vespa.domain.tld"; + + @Test + public void test_signing() throws Exception { + ExtensionsGenerator extGen = new ExtensionsGenerator(); + String subject = "C=NO,OU=Vespa,CN=" + requestersHostname; + PKCS10CertificationRequest request = makeRequest(subject, extGen.generate()); + + X509Certificate certificate = signer.generateX509Certificate(request, requestersHostname); + assertCertificate(certificate, subject, Collections.singleton(Extension.basicConstraints.getId())); + } + + @Test + public void common_name_test() throws Exception { + CertificateSigner.verifyCertificateCommonName( + new X500Name("CN=" + requestersHostname), requestersHostname); + CertificateSigner.verifyCertificateCommonName( + new X500Name("C=NO,OU=Vespa,CN=" + requestersHostname), requestersHostname); + CertificateSigner.verifyCertificateCommonName( + new X500Name("C=NO+OU=org,CN=" + requestersHostname), requestersHostname); + + assertCertificateCommonNameException("C=NO", "Only 1 common name should be set"); + assertCertificateCommonNameException("C=US+CN=abc123.domain.tld,C=NO+CN=" + requestersHostname, "Only 1 common name should be set"); + assertCertificateCommonNameException("CN=evil.hostname.domain.tld", + "Expected common name to be tenant-123.us-north-1.vespa.domain.tld, but was evil.hostname.domain.tld"); + } + + @Test(expected = IllegalArgumentException.class) + public void extensions_test_subject_alternative_names() throws Exception { + ExtensionsGenerator extGen = new ExtensionsGenerator(); + extGen.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(new GeneralName[] { + new GeneralName(GeneralName.dNSName, "some.other.domain.tld")})); + PKCS10CertificationRequest request = makeRequest("OU=Vespa", extGen.generate()); + + CertificateSigner.verifyCertificateExtensions(request); + } + + @Test + public void extensions_allowed() throws Exception { + ExtensionsGenerator extGen = new ExtensionsGenerator(); + extGen.addExtension(Extension.certificateIssuer, true, new byte[0]); + PKCS10CertificationRequest request = makeRequest("OU=Vespa", extGen.generate()); + + CertificateSigner.verifyCertificateExtensions(request); + } + + private void assertCertificateCommonNameException(String subject, String expectedMessage) { + try { + CertificateSigner.verifyCertificateCommonName(new X500Name(subject), requestersHostname); + fail("Expected to fail"); + } catch (IllegalArgumentException e) { + assertEquals(expectedMessage, e.getMessage()); + } + } + + private void assertCertificate(X509Certificate certificate, String expectedSubjectName, Set<String> expectedExtensions) throws Exception { + assertEquals(3, certificate.getVersion()); + assertEquals(BigInteger.valueOf(startTime), certificate.getSerialNumber()); + assertEquals(startTime, certificate.getNotBefore().getTime()); + assertEquals(startTime + CertificateSigner.CERTIFICATE_EXPIRATION.toMillis(), certificate.getNotAfter().getTime()); + assertEquals(CertificateSigner.SIGNER_ALGORITHM, certificate.getSigAlgName()); + assertEquals(expectedSubjectName, certificate.getSubjectDN().getName()); + assertEquals("CN=" + cfgServerHostname, certificate.getIssuerX500Principal().getName()); + + Set<String> extensions = Stream.of(certificate.getNonCriticalExtensionOIDs(), + certificate.getCriticalExtensionOIDs()) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + assertEquals(expectedExtensions, extensions); + + certificate.verify(caKeyPair.getPublic()); + } + + private PKCS10CertificationRequest makeRequest(String subject, Extensions extensions) throws Exception { + PKCS10CertificationRequestBuilder builder = new JcaPKCS10CertificationRequestBuilder( + new X500Name(subject), clientKeyPair.getPublic()); + builder.addAttribute(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions); + + ContentSigner signGen = new JcaContentSignerBuilder(CertificateSigner.SIGNER_ALGORITHM).build(caKeyPair.getPrivate()); + return builder.build(signGen); + } + + private static KeyPair getKeyPair() { + try { + return KeyPairGenerator.getInstance("RSA").genKeyPair(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CsrSerializedPayloadTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CsrSerializedPayloadTest.java new file mode 100644 index 00000000000..b8433856f95 --- /dev/null +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ca/CsrSerializedPayloadTest.java @@ -0,0 +1,32 @@ +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.ca; + +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +/** + * @author bjorncs + */ +public class CsrSerializedPayloadTest { + + @Test + public void it_can_be_deserialized() throws IOException { + String serialized = "{\"csr\":\"-----BEGIN CERTIFICATE REQUEST-----\\nMIICVDCCATwCAQAwDzENMAsGA1UEAwwEdGV" + + "zdDCCASIwDQYJKoZIhvcNAQEBBQAD\\nggEPADCCAQoCggEBAL7xra4De9B54yY6lw8Ka/lt7lDEKQRp42RYzpXjHIQXFgr8" + + "\\n+EvJCLEldFoqfOm728KAWQq/8YdFR4hBwOz8Rr8khJKMBCQ2DWvGYz2705nr3j3v\\nsd3RE5i8n8cUdKiHRuOf305xgy" + + "970TFb+s5/tQOfDMDfvC/BdHNhB4pc0P04CVs/\\nzusKvghdSXFVufAuVaY30ZyviqrDVlBZnI158MmRzfINwP70ZYn5wsq" + + "crKzgSUBp\\nH/WjxaklSzGOH8Uk/EKVx0luzAxtTU8jO7MU1+EG8H4E+FI9ijdjftYyko5UAOQO\\nJGiI9/qHJIMVOIcQa" + + "k1PA5+2/0NbtVxihQi/uJcCAwEAAaAAMA0GCSqGSIb3DQEB\\nCwUAA4IBAQAelFvM6PyDFufv9pNmFigNqOO+r8ats9Xak9" + + "JVtGERo9KFcNDAkawD\\nMPzWQeB87oPnB5dlSdkI2J/jIV7/zR9Qoa2qZlKeL4vUIvfMTj5EOmQLn4ofoBwa\\n50D8Ro3D" + + "06Ohb1KE3seOK2FfVybiATpoaICCjb0ibhx4lNsJGZXpw6F2OdTRi8Fb\\n7kfgLiLPCH+UiHDeVnjVVr/PUKeSImgv44mb4" + + "c6EU29MYkM4LxCY9/c4scG7Pq+s\\nuHU5Tepjsnmkdtip5NzS7csPXENEygKyksPHWFFojPrtF6nFkMzzIPUgKbsmm4+H\\" + + "nfJihCYL3pc3+bVYl87TIcdohJ1GYvfw7\\n-----END CERTIFICATE REQUEST-----\\n\"}"; + CsrSerializedPayload csrSerializedPayload = Utils.getMapper().readValue(serialized, CsrSerializedPayload.class); + assertThat(csrSerializedPayload.csr, notNullValue()); + } + +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index d77757374ce..0c12e137e27 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -1,4 +1,4 @@ -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; import com.google.common.collect.ImmutableSet; @@ -13,10 +13,9 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AthenzInstanceProviderServiceTest.AutoGeneratedKeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AutoGeneratedKeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.ProviderUniqueId; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.SignedIdentityDocument; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -27,8 +26,7 @@ import org.junit.Test; import java.util.HashSet; import java.util.Optional; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.AthenzInstanceProviderServiceTest.getAthenzProviderConfig; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.AthenzInstanceProviderServiceTest.getZoneConfig; +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.TestUtils.getAthenzProviderConfig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.eq; @@ -66,17 +64,9 @@ public class IdentityDocumentGeneratorTest { String dnsSuffix = "vespa.dns.suffix"; AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix, ZONE); - IdentityDocumentGenerator identityDocumentGenerator = new IdentityDocumentGenerator( - config, - getZoneConfig(config, ZONE), - nodeRepository, - ZONE, - keyProvider); - String rawSignedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(hostname); - - - SignedIdentityDocument signedIdentityDocument = - Utils.getMapper().readValue(rawSignedIdentityDocument, SignedIdentityDocument.class); + IdentityDocumentGenerator identityDocumentGenerator = + new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); + SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(hostname); // Verify attributes assertEquals(hostname, signedIdentityDocument.identityDocument.instanceHostname); @@ -92,7 +82,7 @@ public class IdentityDocumentGeneratorTest { // Validate signature assertTrue("Message", InstanceValidator.isSignatureValid(keyProvider.getPublicKey(0), - signedIdentityDocument.rawIdentityDocument, - signedIdentityDocument.signature)); + signedIdentityDocument.rawIdentityDocument, + signedIdentityDocument.signature)); } -}
\ No newline at end of file +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index c1fab319ebf..c68a8805abc 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -1,4 +1,4 @@ -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.model.api.ApplicationInfo; @@ -8,11 +8,12 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AthenzInstanceProviderServiceTest.AutoGeneratedKeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.IdentityDocument; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.ProviderUniqueId; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.model.SignedIdentityDocument; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AutoGeneratedKeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument.IdentityDocument; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument.ProviderUniqueId; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument.SignedIdentityDocument; import org.junit.Test; import java.security.PrivateKey; @@ -28,8 +29,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceValidator.SERVICE_PROPERTIES_DOMAIN_KEY; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceValidator.SERVICE_PROPERTIES_SERVICE_KEY; +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator.SERVICE_PROPERTIES_DOMAIN_KEY; +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator.SERVICE_PROPERTIES_SERVICE_KEY; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -168,4 +169,4 @@ public class InstanceValidatorTest { return new ApplicationInfo(appId, 0, model); } -}
\ No newline at end of file +} diff --git a/bundle-plugin-test/pom.xml b/bundle-plugin-test/pom.xml index 2112d616523..8d8bf48cc07 100644 --- a/bundle-plugin-test/pom.xml +++ b/bundle-plugin-test/pom.xml @@ -10,7 +10,7 @@ <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> </parent> - <groupId>com.yahoo.container.maven.plugin</groupId> + <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin-test</artifactId> <version>6-SNAPSHOT</version> <packaging>container-plugin</packaging> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java deleted file mode 100644 index 7e1d247281c..00000000000 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FileReferenceCreator.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.application.provider; - -import com.yahoo.config.FileReference; - -import java.lang.reflect.Constructor; - -/** - * Convenience for creating a {@link com.yahoo.config.FileReference}. - * - * @author gjoranv - */ -public class FileReferenceCreator { - - public static FileReference create(String stringVal) { - try { - Constructor<FileReference> ctor = FileReference.class.getDeclaredConstructor(String.class); - ctor.setAccessible(true); - return ctor.newInstance(stringVal); - } catch (Exception e) { - throw new RuntimeException("Could not create a new " + FileReference.class.getName() + - ". This should never happen!", e); - } - } - -} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index ca0b37d8cc3..d635fe90ded 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -17,7 +17,7 @@ import java.util.Set; public class MockFileRegistry implements FileRegistry { public FileReference addFile(String relativePath) { - return FileReferenceCreator.create("0123456789abcdef"); + return new FileReference("0123456789abcdef"); } @Override @@ -25,8 +25,8 @@ public class MockFileRegistry implements FileRegistry { return "localhost.fortestingpurposesonly"; } - public static final Entry entry1 = new Entry("component/path1", FileReferenceCreator.create("1234")); - public static final Entry entry2 = new Entry("component/path2", FileReferenceCreator.create("56789")); + public static final Entry entry1 = new Entry("component/path1", new FileReference("1234")); + public static final Entry entry2 = new Entry("component/path2", new FileReference("56789")); public List<Entry> export() { List<Entry> result = new ArrayList<>(); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java index 29e83e00305..0b0b799f47f 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -70,7 +70,7 @@ public class PreGeneratedFileRegistry implements FileRegistry { } public FileReference addFile(String relativePath) { - return FileReferenceCreator.create(path2Hash.get(relativePath)); + return new FileReference(path2Hash.get(relativePath)); } @Override @@ -86,7 +86,7 @@ public class PreGeneratedFileRegistry implements FileRegistry { public List<Entry> export() { List<Entry> entries = new ArrayList<>(); for (Map.Entry<String, String> entry : path2Hash.entrySet()) { - entries.add(new Entry(entry.getKey(), FileReferenceCreator.create(entry.getValue()))); + entries.add(new Entry(entry.getKey(), new FileReference(entry.getValue()))); } return entries; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index f46fe39e8c4..64e986abf9b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -3,6 +3,8 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableList; import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -14,7 +16,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -40,28 +41,36 @@ public class DeploymentSpec { UpgradePolicy.defaultPolicy, Collections.emptyList(), Collections.emptyList(), - "<deployment version='1.0'/>"); + "<deployment version='1.0'/>", + Optional.empty(), + Optional.empty()); private final Optional<String> globalServiceId; private final UpgradePolicy upgradePolicy; private final List<ChangeBlocker> changeBlockers; private final List<Step> steps; private final String xmlForm; + private final Optional<AthenzDomain> athenzDomain; + private final Optional<AthenzService> athenzService; public DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, List<ChangeBlocker> changeBlockers, List<Step> steps) { - this(globalServiceId, upgradePolicy, changeBlockers, steps, null); + this(globalServiceId, upgradePolicy, changeBlockers, steps, null, Optional.empty(), Optional.empty()); } public DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, - List<ChangeBlocker> changeBlockers, List<Step> steps, String xmlForm) { + List<ChangeBlocker> changeBlockers, List<Step> steps, String xmlForm, + Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { validateTotalDelay(steps); this.globalServiceId = globalServiceId; this.upgradePolicy = upgradePolicy; this.changeBlockers = changeBlockers; this.steps = ImmutableList.copyOf(completeSteps(new ArrayList<>(steps))); this.xmlForm = xmlForm; + this.athenzDomain = athenzDomain; + this.athenzService = athenzService; validateZones(this.steps); + validateAthenz(); } /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ @@ -82,7 +91,30 @@ public class DeploymentSpec { for (DeclaredZone zone : step.zones()) ensureUnique(zone, zones); } - + + /* + * Throw an IllegalArgumentException if Athenz configuration violates: + * domain not configured -> no zone can configure service + * domain configured -> all zones must configure service + */ + private void validateAthenz() { + // If athenz domain is not set, athenz service cannot be set on any level + if (! athenzDomain.isPresent()) { + for (DeclaredZone zone : zones()) { + if(zone.athenzService().isPresent()) { + throw new IllegalArgumentException("Athenz service configured for zone: " + zone + ", but Athenz domain is not configured"); + } + } + // if athenz domain is not set, athenz service must be set implicitly or directly on all zones. + } else if(! athenzService.isPresent()) { + for (DeclaredZone zone : zones()) { + if(! zone.athenzService().isPresent()) { + throw new IllegalArgumentException("Athenz domain is configured, but Athenz service not configured for zone: " + zone); + } + } + } + } + private void ensureUnique(DeclaredZone zone, Set<DeclaredZone> zones) { if ( ! zones.add(zone)) throw new IllegalArgumentException(zone + " is listed twice in deployment.xml"); @@ -90,9 +122,6 @@ public class DeploymentSpec { /** Adds missing required steps and reorders steps to a permissible order */ private static List<Step> completeSteps(List<Step> steps) { - // Ensure no duplicate deployments to the same zone - steps = new ArrayList<>(new LinkedHashSet<>(steps)); - // Add staging if required and missing if (steps.stream().anyMatch(step -> step.deploysTo(Environment.prod)) && steps.stream().noneMatch(step -> step.deploysTo(Environment.staging))) { @@ -216,6 +245,21 @@ public class DeploymentSpec { return b.toString(); } + /** Returns the athenz domain if configured */ + public Optional<AthenzDomain> athenzDomain() { + return athenzDomain; + } + + /** Returns the athenz service for environment/region if configured */ + public Optional<AthenzService> athenzService(Environment environment, RegionName region) { + AthenzService athenzService = zones().stream() + .filter(zone -> zone.deploysTo(environment, Optional.of(region))) + .findFirst() + .flatMap(DeclaredZone::athenzService) + .orElse(this.athenzService.orElse(null)); + return Optional.ofNullable(athenzService); + } + /** This may be invoked by a continuous build */ public static void main(String[] args) { if (args.length != 2 && args.length != 3) { @@ -280,11 +324,17 @@ public class DeploymentSpec { private final boolean active; + private Optional<AthenzService> athenzService; + public DeclaredZone(Environment environment) { this(environment, Optional.empty(), false); } public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active) { + this(environment, region, active, Optional.empty()); + } + + public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active, Optional<AthenzService> athenzService) { if (environment != Environment.prod && region.isPresent()) throw new IllegalArgumentException("Non-prod environments cannot specify a region"); if (environment == Environment.prod && ! region.isPresent()) @@ -292,6 +342,7 @@ public class DeploymentSpec { this.environment = environment; this.region = region; this.active = active; + this.athenzService = athenzService; } public Environment environment() { return environment; } @@ -302,6 +353,8 @@ public class DeploymentSpec { /** Returns whether this zone should receive production traffic */ public boolean active() { return active; } + public Optional<AthenzService> athenzService() { return athenzService; } + @Override public List<DeclaredZone> zones() { return Collections.singletonList(this); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 5599b257b16..8bc4e0026a6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -7,6 +7,8 @@ import com.yahoo.config.application.api.DeploymentSpec.Delay; import com.yahoo.config.application.api.DeploymentSpec.ParallelZones; import com.yahoo.config.application.api.DeploymentSpec.Step; import com.yahoo.config.application.api.TimeWindow; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.io.IOUtils; @@ -72,6 +74,7 @@ public class DeploymentSpecXmlReader { if (environment == Environment.prod) { for (Element stepTag : XML.getChildren(environmentTag)) { + Optional<AthenzService> athenzService = stringAttribute("athenz-service", environmentTag).map(AthenzService::from); if (stepTag.getTagName().equals("delay")) { steps.add(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + longAttribute("minutes", stepTag) * 60 + @@ -79,11 +82,11 @@ public class DeploymentSpecXmlReader { } else if (stepTag.getTagName().equals("parallel")) { List<DeclaredZone> zones = new ArrayList<>(); for (Element regionTag : XML.getChildren(stepTag)) { - zones.add(readDeclaredZone(environment, regionTag)); + zones.add(readDeclaredZone(environment, athenzService, regionTag)); } steps.add(new ParallelZones(zones)); } else { // a region: deploy step - steps.add(readDeclaredZone(environment, stepTag)); + steps.add(readDeclaredZone(environment, athenzService, stepTag)); } } } else { @@ -94,8 +97,11 @@ public class DeploymentSpecXmlReader { globalServiceId = readGlobalServiceId(environmentTag); else if (readGlobalServiceId(environmentTag).isPresent()) throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag."); + } - return new DeploymentSpec(globalServiceId, readUpgradePolicy(root), readChangeBlockers(root), steps, xmlForm); + Optional<AthenzDomain> athenzDomain = stringAttribute("athenz-domain", root).map(AthenzDomain::from); + Optional<AthenzService> athenzService = stringAttribute("athenz-service", root).map(AthenzService::from); + return new DeploymentSpec(globalServiceId, readUpgradePolicy(root), readChangeBlockers(root), steps, xmlForm, athenzDomain, athenzService); } /** Imposes some constraints on tag order which are not expressible in the schema */ @@ -132,13 +138,19 @@ public class DeploymentSpecXmlReader { } } + /** Returns the given attribute as a string, or Optional.empty if it is not present or empty */ + private Optional<String> stringAttribute(String attributeName, Element tag) { + String value = tag.getAttribute(attributeName); + return Optional.ofNullable(value).filter(s -> ! s.equals("")); + } + private boolean isEnvironmentName(String tagName) { return tagName.equals(testTag) || tagName.equals(stagingTag) || tagName.equals(prodTag); } - private DeclaredZone readDeclaredZone(Environment environment, Element regionTag) { + private DeclaredZone readDeclaredZone(Environment environment, Optional<AthenzService> athenzService, Element regionTag) { return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())), - readActive(regionTag)); + readActive(regionTag), athenzService); } private Optional<String> readGlobalServiceId(Element environmentTag) { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java index a5d54fb84b3..990bce539ba 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java @@ -18,12 +18,15 @@ public interface FileDistribution { void sendDeployedFiles(String hostName, Set<FileReference> fileReferences); void reloadDeployFileDistributor(); - // TODO: Remove when 6.150 is the oldest version used - void limitSendingOfDeployedFilesTo(Collection<String> hostNames); + void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames); + static String getDefaultFileDBRoot() { + return Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"); + } + static File getDefaultFileDBPath() { - return new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")); + return new File(getDefaultFileDBRoot()); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 5b79415c132..521e72ae580 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -6,10 +6,12 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import java.io.File; +import java.net.URI; import java.util.List; import java.util.Optional; import java.util.Set; @@ -41,9 +43,18 @@ public interface ModelContext { boolean multitenant(); ApplicationId applicationId(); List<ConfigServerSpec> configServerSpecs(); + HostName loadBalancerName(); boolean hostedVespa(); Zone zone(); Set<Rotation> rotations(); + + /* + * DEPRECATED + * TODO: Remove when 6.172 and earlier are no longer in use + */ + default URI loadBalancerAddress() { + return URI.create("http://localhost"); + } } } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 8bab2f83448..103724744de 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -97,12 +97,10 @@ public class DeploymentSpecTest { StringReader r = new StringReader( "<deployment version='1.0'>" + " <test/>" + - " <test/>" + " <staging/>" + " <prod>" + " <region active='false'>us-east1</region>" + - " <region active='false'>us-east1</region>" + - " <delay hours='3' minutes='30'/>" + + " <delay hours='3' minutes='30'/>" + " <region active='true'>us-west1</region>" + " </prod>" + "</deployment>" @@ -369,4 +367,56 @@ public class DeploymentSpecTest { assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); } + @Test + public void athenz_config_is_read_from_deployment() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.athenzDomain().get().value(), "domain"); + assertEquals(spec.athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "service"); + } + + @Test + public void athenz_service_is_overridden_from_environment() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>\n" + + " <test/>\n" + + " <prod athenz-service='prod-service'>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.athenzDomain().get().value(), "domain"); + assertEquals(spec.athenzService(Environment.prod, RegionName.from("us-west-1")).get().value(), "prod-service"); + } + + @Test(expected = IllegalArgumentException.class) + public void it_fails_when_athenz_service_is_not_defined() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain'>\n" + + " <prod>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } + + @Test(expected = IllegalArgumentException.class) + public void it_fails_when_athenz_service_is_configured_but_not_athenz_domain() { + StringReader r = new StringReader( + "<deployment>\n" + + " <prod athenz-service='service'>\n" + + " <region active='true'>us-west-1</region>\n" + + " </prod>\n" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + } } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 4e7af809ce7..a4bbba0e6e8 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -240,11 +240,6 @@ <artifactId>filedistributionmanager</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistribution</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> diff --git a/config-model/pom.xml b/config-model/pom.xml index 4e79da3279d..0fdc09e1a61 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -276,11 +276,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistribution</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> <version>${project.version}</version> </dependency> diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java index aec4c5b3ec6..3e96b225226 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java @@ -2,7 +2,10 @@ package com.yahoo.config.model.deploy; import com.yahoo.config.model.api.ConfigServerSpec; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; import java.util.ArrayList; import java.util.List; @@ -18,6 +21,7 @@ public class DeployProperties { private final boolean multitenant; private final ApplicationId applicationId; private final List<ConfigServerSpec> serverSpecs = new ArrayList<>(); + private final HostName loadBalancerName; private final boolean hostedVespa; private final Version vespaVersion; private final Zone zone; @@ -25,7 +29,11 @@ public class DeployProperties { private DeployProperties(boolean multitenant, ApplicationId applicationId, List<ConfigServerSpec> configServerSpecs, - boolean hostedVespa, Version vespaVersion, Zone zone) { + HostName loadBalancerName, + boolean hostedVespa, + Version vespaVersion, + Zone zone) { + this.loadBalancerName = loadBalancerName; this.vespaVersion = vespaVersion; this.zone = zone; this.multitenant = multitenant || hostedVespa || Boolean.getBoolean("multitenant"); @@ -47,6 +55,10 @@ public class DeployProperties { return serverSpecs; } + public HostName loadBalancerName() { + return loadBalancerName; + } + public boolean hostedVespa() { return hostedVespa; } @@ -63,6 +75,7 @@ public class DeployProperties { private ApplicationId applicationId = ApplicationId.defaultId(); private boolean multitenant = false; private List<ConfigServerSpec> configServerSpecs = new ArrayList<>(); + private HostName loadBalancerName; private boolean hostedVespa = false; private Version vespaVersion = Version.fromIntValues(1, 0, 0); private Zone zone = Zone.defaultZone(); @@ -82,6 +95,11 @@ public class DeployProperties { return this; } + public Builder loadBalancerName(HostName loadBalancerName) { + this.loadBalancerName = loadBalancerName; + return this; + } + public Builder vespaVersion(Version version) { this.vespaVersion = version; return this; @@ -98,7 +116,7 @@ public class DeployProperties { } public DeployProperties build() { - return new DeployProperties(multitenant, applicationId, configServerSpecs, hostedVespa, vespaVersion, zone); + return new DeployProperties(multitenant, applicationId, configServerSpecs, loadBalancerName, hostedVespa, vespaVersion, zone); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java index b2cd8574076..c75864f81b7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java @@ -21,6 +21,8 @@ import java.util.Map; */ class ConstantTensorTransformer extends ExpressionTransformer { + public static final String CONSTANT = "constant"; + private final Map<String, Value> constants; private final Map<String, String> rankPropertiesOutput; @@ -64,7 +66,7 @@ class ConstantTensorTransformer extends ExpressionTransformer { return node; } TensorValue tensorValue = (TensorValue)value; - String featureName = "constant(" + node.getName() + ")"; + String featureName = CONSTANT + "(" + node.getName() + ")"; String tensorType = tensorValue.asTensor().type().toString(); rankPropertiesOutput.put(featureName + ".value", tensorValue.toString()); rankPropertiesOutput.put(featureName + ".type", tensorType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index fd249956d5a..1021227b0e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -706,6 +706,7 @@ public class RankProfile implements Serializable, Cloneable { expression = new ConstantTensorTransformer(constants, rankPropertiesOutput).transform(expression); expression = new MacroInliner(inlineMacros).transform(expression); expression = new MacroShadower(getMacros()).transform(expression); + expression = new TensorTransformer(this).transform(expression); expression = new Simplifier().transform(expression); for (Map.Entry<String, String> rankProperty : rankPropertiesOutput.entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java new file mode 100644 index 00000000000..69e353ceb35 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/TensorTransformer.java @@ -0,0 +1,290 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Reduce; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Transforms and simplifies tensor expressions. + * + * Currently transforms min(tensor,dim) and max(tensor,dim) to + * reduce(tensor,min/max,dim). This is necessary as the backend does + * not recognize these forms of min and max. + * + * @author lesters + */ +public class TensorTransformer extends ExpressionTransformer { + + private Search search; + private RankProfile rankprofile; + private Map<String, RankProfile.Macro> macros; + + public TensorTransformer(RankProfile rankprofile) { + this.rankprofile = rankprofile; + this.search = rankprofile.getSearch(); + this.macros = rankprofile.getMacros(); + } + + @Override + public ExpressionNode transform(ExpressionNode node) { + if (node instanceof CompositeNode) { + node = transformChildren((CompositeNode) node); + } + if (node instanceof FunctionNode) { + node = transformFunctionNode((FunctionNode) node); + } + return node; + } + + private ExpressionNode transformFunctionNode(FunctionNode node) { + switch (node.getFunction()) { + case min: + case max: + return transformMaxAndMinFunctionNode(node); + } + return node; + } + + /** + * Transforms max and min functions if it can be proven that the first + * argument resolves to a tensor and the second argument is a valid + * dimension in the tensor. If these do not hold, the node will not + * be transformed. + * + * The test for whether or not the first argument resolves to a tensor + * is to evaluate that expression. All values used in the expression + * is bound to a context with dummy values with enough information to + * deduce tensor types. + * + * There is currently no guarantee that all cases will be found. For + * instance, if-statements are problematic. + */ + private ExpressionNode transformMaxAndMinFunctionNode(FunctionNode node) { + if (node.children().size() != 2) { + return node; + } + ExpressionNode arg1 = node.children().get(0); + Optional<String> dimension = dimensionName(node.children().get(1)); + if (dimension.isPresent()) { + try { + Context context = buildContext(arg1); + Value value = arg1.evaluate(context); + if (isTensorWithDimension(value, dimension.get())) { + return replaceMaxAndMinFunction(node); + } + } catch (IllegalArgumentException e) { + // Thrown from evaluate if some variables are not bound, for + // instance for a backend rank feature. Means we don't have + // enough information to replace expression. + } + } + return node; + } + + private Optional<String> dimensionName(ExpressionNode arg) { + if (arg instanceof ReferenceNode && ((ReferenceNode)arg).children().size() == 0) { + return Optional.of(((ReferenceNode) arg).getName()); + } + return Optional.empty(); + } + + private boolean isTensorWithDimension(Value value, String dimension) { + if (value instanceof TensorValue) { + Tensor tensor = ((TensorValue) value).asTensor(); + TensorType type = tensor.type(); + return type.dimensionNames().contains(dimension); + } + return false; + } + + private ExpressionNode replaceMaxAndMinFunction(FunctionNode node) { + ExpressionNode arg1 = node.children().get(0); + ExpressionNode arg2 = node.children().get(1); + + TensorFunctionNode.TensorFunctionExpressionNode expression = TensorFunctionNode.wrapArgument(arg1); + Reduce.Aggregator aggregator = Reduce.Aggregator.valueOf(node.getFunction().name()); + String dimension = ((ReferenceNode) arg2).getName(); + + return new TensorFunctionNode(new Reduce(expression, aggregator, dimension)); + } + + /** + * Creates an evaluation context by iterating through the expression tree, and + * adding dummy values with correct types to the context. + */ + private Context buildContext(ExpressionNode node) { + Context context = new MapContext(); + addRoot(node, context); + return context; + } + + private Value emptyStringValue() { + return new StringValue(""); + } + + private Value emptyDoubleValue() { + return new DoubleValue(0.0); + } + + private Value emptyTensorValue(TensorType type) { + Tensor empty = Tensor.Builder.of(type).build(); + return new TensorValue(empty); + } + + private void addRoot(ExpressionNode node, Context context) { + addChildren(node, context); + if (node instanceof ReferenceNode) { + ReferenceNode referenceNode = (ReferenceNode) node; + addIfAttribute(referenceNode, context); + addIfConstant(referenceNode, context); + addIfQuery(referenceNode, context); + addIfTensorFrom(referenceNode, context); + addIfMacro(referenceNode, context); + } + } + + private void addChildren(ExpressionNode node, Context context) { + if (node instanceof CompositeNode) { + List<ExpressionNode> children = ((CompositeNode) node).children(); + for (ExpressionNode child : children) { + addRoot(child, context); + } + } + } + + private void addIfAttribute(ReferenceNode node, Context context) { + if (!node.getName().equals("attribute")) { + return; + } + if (node.children().size() == 0) { + return; + } + String attribute = node.children().get(0).toString(); + Attribute a = search.getAttribute(attribute); + if (a == null) { + return; + } + Value v; + if (a.getType() == Attribute.Type.STRING) { + v = emptyStringValue(); + } else if (a.getType() == Attribute.Type.TENSOR) { + v = emptyTensorValue(a.tensorType().orElseThrow(RuntimeException::new)); + } else { + v = emptyDoubleValue(); + } + context.put(node.toString(), v); + } + + private void addIfConstant(ReferenceNode node, Context context) { + if (!node.getName().equals(ConstantTensorTransformer.CONSTANT)) { + return; + } + if (node.children().size() != 1) { + return; + } + ExpressionNode child = node.children().get(0); + while (child instanceof CompositeNode && ((CompositeNode) child).children().size() > 0) { + child = ((CompositeNode) child).children().get(0); + } + String name = child.toString(); + addIfConstantInRankProfile(name, node, context); + addIfConstantInRankingConstants(name, node, context); + } + + private void addIfConstantInRankProfile(String name, ReferenceNode node, Context context) { + if (rankprofile.getConstants().containsKey(name)) { + context.put(node.toString(), rankprofile.getConstants().get(name)); + } + } + + private void addIfConstantInRankingConstants(String name, ReferenceNode node, Context context) { + for (RankingConstant rankingConstant : search.getRankingConstants()) { + if (rankingConstant.getName().equals(name)) { + context.put(node.toString(), emptyTensorValue(rankingConstant.getTensorType())); + } + } + } + + private void addIfQuery(ReferenceNode node, Context context) { + if (!node.getName().equals("query")) { + return; + } + if (node.children().size() != 1) { + return; + } + String name = node.children().get(0).toString(); + if (rankprofile.getQueryFeatureTypes().containsKey(name)) { + String type = rankprofile.getQueryFeatureTypes().get(name); + Value v; + if (type.contains("tensor")) { + v = emptyTensorValue(TensorType.fromSpec(type)); + } else if (type.equalsIgnoreCase("string")) { + v = emptyStringValue(); + } else { + v = emptyDoubleValue(); + } + context.put(node.toString(), v); + } + } + + private void addIfTensorFrom(ReferenceNode node, Context context) { + if (!node.getName().startsWith("tensorFrom")) { + return; + } + if (node.children().size() < 1 || node.children().size() > 2) { + return; + } + ExpressionNode source = node.children().get(0); + if (source instanceof CompositeNode && ((CompositeNode) source).children().size() > 0) { + source = ((CompositeNode) source).children().get(0); + } + String dimension = source.toString(); + if (node.children().size() == 2) { + dimension = node.children().get(1).toString(); + } + TensorType type = (new TensorType.Builder()).mapped(dimension).build(); + context.put(node.toString(), emptyTensorValue(type)); + } + + private void addIfMacro(ReferenceNode node, Context context) { + RankProfile.Macro macro = macros.get(node.getName()); + if (macro == null) { + return; + } + ExpressionNode root = macro.getRankingExpression().getRoot(); + Context macroContext = buildContext(root); + addMacroArguments(node, context, macro, macroContext); + Value value = root.evaluate(macroContext); + context.put(node.toString(), value); + } + + private void addMacroArguments(ReferenceNode node, Context context, RankProfile.Macro macro, Context macroContext) { + if (macro.getFormalParams().size() > 0 && node.children().size() > 0) { + for (int i = 0; i < macro.getFormalParams().size() && i < node.children().size(); ++i) { + String param = macro.getFormalParams().get(i); + ExpressionNode argumentExpression = node.children().get(i); + Value arg = argumentExpression.evaluate(context); + macroContext.put(param, arg); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Logd.java b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java index f15c60ca3c0..0f7418582a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Logd.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; +import com.yahoo.cloud.config.log.LogdConfig; + /** * There is one logd running on each Vespa host, and one instance of * this class is therefore created by each instance of class {@link @@ -8,7 +10,10 @@ package com.yahoo.vespa.model; * * @author gjoranv */ -public class Logd extends AbstractService { +public class Logd + extends AbstractService + implements LogdConfig.Producer +{ /** * Creates a new Logd instance. @@ -17,18 +22,28 @@ public class Logd extends AbstractService { super(host, "logd"); setProp("clustertype", "hosts"); setProp("clustername", "admin"); + portsMeta.on(0).tag("http").tag("state"); } /** - * Logd does not need any ports. + * Logd needs a state port. * * @return The number of ports reserved by the logd */ - public int getPortCount() { return 0; } + public int getPortCount() { return 1; } + + /** Returns the desired base port for this service. */ + public int getWantedPort() { return 19089; } - /** + /** * @return The command used to start logd */ public String getStartupCommand() { return "exec sbin/vespa-logd"; } + @Override + public int getHealthPort() { return getRelativePort(0); } + + public void getConfig(LogdConfig.Builder builder) { + builder.stateport(getHealthPort()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index fc27f9e8dc7..74512e70ebe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -147,6 +147,7 @@ public class VespaModelFactory implements ModelFactory { return new DeployProperties.Builder() .applicationId(properties.applicationId()) .configServerSpecs(properties.configServerSpecs()) + .loadBalancerName(properties.loadBalancerName()) .multitenant(properties.multitenant()) .hostedVespa(properties.hostedVespa()) .vespaVersion(getVersion()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java index b58406e1935..0160978773d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java @@ -7,28 +7,39 @@ import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; /** * Options for controlling the behavior of the file distribution services. + * * @author tonytv */ public class FileDistributionOptions implements FiledistributorConfig.Producer { + public static FileDistributionOptions defaultOptions() { return new FileDistributionOptions(); } - private FileDistributionOptions() {} + private FileDistributionOptions() { + } + + private BinaryScaledAmount uploadBitRate = new BinaryScaledAmount(); + private BinaryScaledAmount downloadBitRate = new BinaryScaledAmount(); + private boolean disabled = false; - private BinaryScaledAmount uploadbitrate = new BinaryScaledAmount(); - private BinaryScaledAmount downloadbitrate = new BinaryScaledAmount(); - //Called through reflection - public void downloadbitrate(BinaryScaledAmount amount) { + public void downloadBitRate(BinaryScaledAmount amount) { ensureNonNegative(amount); - downloadbitrate = amount; + downloadBitRate = amount; } - //Called through reflection - public void uploadbitrate(BinaryScaledAmount amount) { + public void uploadBitRate(BinaryScaledAmount amount) { ensureNonNegative(amount); - uploadbitrate = amount; + uploadBitRate = amount; + } + + public void disabled(boolean value) { + disabled = value; + } + + public boolean disabled() { + return disabled; } private void ensureNonNegative(BinaryScaledAmount amount) { @@ -38,12 +49,12 @@ public class FileDistributionOptions implements FiledistributorConfig.Producer { private int byteRate(BinaryScaledAmount bitRate) { BinaryScaledAmount byteRate = bitRate.divide(8); - return (int)byteRate.as(BinaryPrefix.unit); + return (int) byteRate.as(BinaryPrefix.unit); } @Override public void getConfig(FiledistributorConfig.Builder builder) { - builder.maxuploadspeed((double)byteRate(uploadbitrate)); - builder.maxdownloadspeed((double)byteRate(downloadbitrate)); + builder.maxuploadspeed((double) byteRate(uploadBitRate)); + builder.maxdownloadspeed((double) byteRate(downloadBitRate)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f5e3e64bac3..2dcf874a66e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -214,6 +214,9 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_fragmentation_factor.average")); metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_fragmentation_factor.average")); metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_fragmentation_factor.average")); + metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_limit.last")); + metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_limit.last")); + metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_limit.last")); // resource usage metrics.add(new Metric("content.proton.resource_usage.disk.average")); @@ -303,11 +306,13 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.spi.put.success.average")); metrics.add(new Metric("vds.filestor.spi.remove.success.average")); metrics.add(new Metric("vds.filestor.spi.update.success.average")); + metrics.add(new Metric("vds.filestor.spi.deleteBucket.success.average")); metrics.add(new Metric("vds.filestor.spi.get.success.average")); metrics.add(new Metric("vds.filestor.spi.iterate.success.average")); metrics.add(new Metric("vds.filestor.spi.put.success.rate")); metrics.add(new Metric("vds.filestor.spi.remove.success.rate")); metrics.add(new Metric("vds.filestor.spi.update.success.rate")); + metrics.add(new Metric("vds.filestor.spi.deleteBucket.success.rate")); metrics.add(new Metric("vds.filestor.spi.get.success.rate")); metrics.add(new Metric("vds.filestor.spi.iterate.success.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java index 55c3f569900..8a5d6846a64 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java @@ -6,47 +6,42 @@ import com.yahoo.text.XML; import com.yahoo.vespa.model.admin.FileDistributionOptions; import org.w3c.dom.Element; +import java.util.Optional; + /** - * Builds a file distribution options. - * @author tonytv + * Builds file distribution options. + * + * @author Tony Vaagenes + * @author hmusum */ public class DomFileDistributionOptionsBuilder { + private static void throwExceptionForElementInFileDistribution(String subElement, String reason) { throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason); } - private static void callSetter(FileDistributionOptions options, String name, BinaryScaledAmount amount) { - try { - options.getClass().getMethod(name, BinaryScaledAmountParser.class).invoke(options, amount); - } catch (IllegalArgumentException e) { - throwExceptionForElementInFileDistribution(name, e.getMessage()); - } - catch (Exception e) { - if (e instanceof RuntimeException) - throw (RuntimeException)e; - else - throw new RuntimeException(e); - } - } - - private static void setIfPresent(FileDistributionOptions options, String name, Element fileDistributionElement) { + private Optional<BinaryScaledAmount> getAmount(String name, Element fileDistributionElement) { + Element optionElement = XML.getChild(fileDistributionElement, name); try { - Element optionElement = XML.getChild(fileDistributionElement, name); if (optionElement != null) { String valueString = XML.getValue(optionElement); - BinaryScaledAmount amount = BinaryScaledAmountParser.parse(valueString); - callSetter(options, name, amount); + return Optional.of(BinaryScaledAmountParser.parse(valueString)); } } catch (NumberFormatException e) { throwExceptionForElementInFileDistribution(name, "Expected a valid number. (Message = " + e.getMessage() + ")."); } + return Optional.empty(); } public FileDistributionOptions build(Element fileDistributionElement) { FileDistributionOptions options = FileDistributionOptions.defaultOptions(); if (fileDistributionElement != null) { - setIfPresent(options, "uploadbitrate", fileDistributionElement); - setIfPresent(options, "downloadbitrate", fileDistributionElement); + getAmount("uploadbitrate", fileDistributionElement).ifPresent(options::uploadBitRate); + getAmount("downloadbitrate", fileDistributionElement).ifPresent(options::downloadBitRate); + Element disable = XML.getChild(fileDistributionElement, "disabled"); + if (disable != null) { + options.disabled(Boolean.valueOf(XML.getValue(disable))); + } } return options; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 5915f0cea0b..8991bfa6215 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -201,7 +201,7 @@ public class Container extends AbstractService implements } private void initDefaultJettyConnector() { - defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", getSearchPort(), null)); + defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", getSearchPort())); } private boolean hasDocproc() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 91d5b7fe267..4383e55e45d 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -9,13 +9,10 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; -import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; @@ -85,7 +82,6 @@ import com.yahoo.vespaclient.config.FeederConfig; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.Reader; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -382,8 +378,6 @@ public final class ContainerCluster container.setClusterName(name); container.setProp("clustername", name) .setProp("index", this.containers.size()); - setRotations(container, getRotations(), getGlobalServiceId(), name); - container.setProp("activeRotation", Boolean.toString(getActiveRotation())); containers.add(container); } @@ -393,55 +387,6 @@ public final class ContainerCluster } } - private Optional<String> getGlobalServiceId() { - Optional<DeploymentSpec> deploymentSpec = getDeploymentSpec(); - if (deploymentSpec.isPresent()) return deploymentSpec.get().globalServiceId(); - return Optional.empty(); - } - - private Set<Rotation> getRotations() { - return Optional.ofNullable(getRoot()) - .map(root -> root.getDeployState().getRotations()) - .orElse(Collections.emptySet()); - } - - private boolean getActiveRotation() { - return Optional.ofNullable(getRoot()) - .map(root -> root.getDeployState().getProperties().zone()) - .map(this::zoneHasActiveRotation) - .orElse(false); - } - - private boolean zoneHasActiveRotation(Zone zone) { - Optional<DeploymentSpec> spec = getDeploymentSpec(); - if (!spec.isPresent()) { - return false; - } - return spec.get().zones().stream() - .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && - declaredZone.active()); - } - - private Optional<DeploymentSpec> getDeploymentSpec() { - Optional<DeploymentSpec> deploymentSpec = Optional.empty(); - AbstractConfigProducerRoot root = getRoot(); - if (root != null) { - final Optional<Reader> deployment = root.getDeployState().getApplicationPackage().getDeployment(); - if (deployment.isPresent()) { - deploymentSpec = Optional.of(DeploymentSpec.fromXml(deployment.get())); - } - } - return deploymentSpec; - } - - private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) { - if ( ! rotations.isEmpty() && globalServiceId.isPresent()) { - if (containerClusterName.equals(globalServiceId.get())) { - container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(","))); - } - } - } - public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) { if (this.processingChains != null) throw new IllegalStateException("ProcessingChains should only be set once."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Identity.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Identity.java deleted file mode 100644 index 81762f95a90..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Identity.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container; - -import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl; -import com.yahoo.vespa.model.container.component.SimpleComponent; - -/** - * @author mortent - */ -public class Identity extends SimpleComponent implements IdentityConfig.Producer { - public static final String CLASS = AthenzIdentityProviderImpl.class.getName(); - - private final String domain; - private final String service; - private final String loadBalancerAddress; - - public Identity(String domain, String service, String loadBalancerAddress) { - super(CLASS); - this.domain = domain; - this.service = service; - this.loadBalancerAddress = loadBalancerAddress; - } - - @Override - public void getConfig(IdentityConfig.Builder builder) { - builder.domain(domain); - builder.service(service); - builder.loadBalancerAddress(loadBalancerAddress); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java new file mode 100644 index 00000000000..0368f7eaf3e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.HostName; +import com.yahoo.container.core.identity.IdentityConfig; +import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +/** + * @author mortent + */ +public class IdentityProvider extends SimpleComponent implements IdentityConfig.Producer { + public static final String CLASS = AthenzIdentityProviderImpl.class.getName(); + + private final AthenzDomain domain; + private final AthenzService service; + private final HostName loadBalancerName; + + public IdentityProvider(AthenzDomain domain, AthenzService service, HostName loadBalancerName) { + super(CLASS); + this.domain = domain; + this.service = service; + this.loadBalancerName = loadBalancerName; + } + + @Override + public void getConfig(IdentityConfig.Builder builder) { + builder.domain(domain.value()); + builder.service(service.value()); + // Current interpretation of loadbalancer address is: hostname. + // Config should be renamed or send the uri + builder.loadBalancerAddress(loadBalancerName.value()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index 9d3d8b32ddb..22c42056b3f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -13,31 +14,50 @@ import static com.yahoo.component.ComponentSpecification.fromString; import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.21.0 + * @author Einar M R Rosenvinge + * @author bjorncs */ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig.Producer { private final String name; - private volatile int listenPort; + private final int listenPort; private final Element legacyConfig; - public ConnectorFactory(final String name, final int listenPort, final Element legacyConfig) { + public ConnectorFactory(String name, int listenPort) { + this(name, listenPort, null, null); + } + + public ConnectorFactory(String name, int listenPort, Element legacyConfig, Element sslKeystoreConfigurator) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(name), fromString("com.yahoo.jdisc.http.server.jetty.ConnectorFactory"), - fromString("jdisc_http_service")) - - )); - - + fromString("jdisc_http_service")))); this.name = name; this.listenPort = listenPort; this.legacyConfig = legacyConfig; + SimpleComponent sslKeyStoreConfigurator = getSslKeyStoreConfigurator(name, sslKeystoreConfigurator); + addChild(sslKeyStoreConfigurator); + inject(sslKeyStoreConfigurator); } @Override public void getConfig(ConnectorConfig.Builder connectorBuilder) { + configureWithLegacyHttpConfig(legacyConfig, connectorBuilder); + connectorBuilder.listenPort(listenPort); + connectorBuilder.name(name); + } + + public String getName() { + return name; + } + + public int getListenPort() { + return listenPort; + } + + // TODO Remove support for legacy config in Vespa 7 + @Deprecated + private static void configureWithLegacyHttpConfig(Element legacyConfig, ConnectorConfig.Builder connectorBuilder) { if (legacyConfig != null) { { Element tcpKeepAliveEnabled = XML.getChild(legacyConfig, "tcpKeepAliveEnabled"); @@ -85,9 +105,7 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig Element ssl = XML.getChild(legacyConfig, "ssl"); Element sslEnabled = XML.getChild(ssl, "enabled"); - if (ssl != null && - sslEnabled != null && - Boolean.parseBoolean(XML.getValue(sslEnabled).trim())) { + if (ssl != null && sslEnabled != null && Boolean.parseBoolean(XML.getValue(sslEnabled).trim())) { ConnectorConfig.Ssl.Builder sslBuilder = new ConnectorConfig.Ssl.Builder(); sslBuilder.enabled(true); { @@ -129,21 +147,18 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig connectorBuilder.ssl(sslBuilder); } } - - connectorBuilder.listenPort(listenPort); - connectorBuilder.name(name); - } - - public String getName() { - return name; } - public int getListenPort() { - return listenPort; - } - - public void setListenPort(int httpPort) { - this.listenPort = httpPort; + private static SimpleComponent getSslKeyStoreConfigurator(String name, Element sslKeystoreConfigurator) { + String idSpec = "ssl-keystore-configurator@" + name; + if (sslKeystoreConfigurator != null) { + String className = sslKeystoreConfigurator.getAttribute("class"); + String bundleName = sslKeystoreConfigurator.getAttribute("bundle"); + return new SimpleComponent(new ComponentModel(idSpec, className, bundleName)); + } else { + return new SimpleComponent( + new ComponentModel(idSpec, DefaultSslKeyStoreConfigurator.class.getName(), "jdisc_http_service")); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index 6271ff817bb..f2012a609a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -33,6 +33,8 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil legacyServerConfig = null; } } - return new ConnectorFactory(name, port, legacyServerConfig); + Element sslKeystoreConfigurator = XML.getChild(serverSpec, "ssl-keystore-configurator"); + return new ConnectorFactory(name, port, legacyServerConfig, sslKeystoreConfigurator); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index c9cc51af867..6efd2f5b9a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -1,20 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.application.Xml; import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.application.api.ApplicationPackage; - -import com.yahoo.path.Path; -import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; - +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; -import org.w3c.dom.Node; - -import java.util.List; /** * Builds the config model for the standalone config server. @@ -24,44 +15,18 @@ import java.util.List; public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { private final CloudConfigOptions options; - private final static String HOSTED_VESPA_INCLUDE_DIR = "hosted-vespa"; public ConfigServerContainerModelBuilder(CloudConfigOptions options) { super(true, Networking.enable); this.options = options; } - @Override public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) { - ApplicationPackage app = modelContext.getDeployState().getApplicationPackage(); - if ( ! app.getFiles(Path.fromString(HOSTED_VESPA_INCLUDE_DIR), ".xml").isEmpty()) { - app.validateIncludeDir(HOSTED_VESPA_INCLUDE_DIR); - List<Element> configModelElements = Xml.allElemsFromPath(app, HOSTED_VESPA_INCLUDE_DIR); - mergeInto(spec, configModelElements); - } - - ConfigserverCluster cluster = new ConfigserverCluster(modelContext.getParentProducer(), "configserver", options); + ConfigserverCluster cluster = new ConfigserverCluster(modelContext.getParentProducer(), "configserver", + options); super.doBuild(model, spec, modelContext.withParent(cluster)); cluster.setContainerCluster(model.getCluster()); } - private void mergeInto(Element destination, List<Element> configModelElements) { - for (Element jdiscElement: configModelElements) { - for (Node child = jdiscElement.getFirstChild(); child != null; child = child.getNextSibling()) { - Node copiedNode = destination.getOwnerDocument().importNode(child, true); - destination.appendChild(copiedNode); - } - } - } - - @Override - protected void addDefaultComponents(ContainerCluster containerCluster) { - // To avoid search specific stuff. - } - - @Override - protected void addDefaultHandlers(ContainerCluster containerCluster) { - addDefaultHandlersExceptStatus(containerCluster); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ce9d0ed27f1..81fc464327e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -6,17 +6,22 @@ import com.yahoo.component.Version; import com.yahoo.config.application.Xml; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.text.XML; @@ -38,7 +43,7 @@ import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.Identity; +import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; @@ -63,7 +68,9 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -93,6 +100,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName(); private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName(); + private static final Logger logger = Logger.getLogger(ContainerModelBuilder.class.getName()); + public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) { super(ContainerModel.class); this.standaloneBuilder = standaloneBuilder; @@ -163,11 +172,42 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Athenz copper argos // NOTE: Must be done after addNodes() - addIdentity(spec, cluster, context.getDeployState().getProperties().configServerSpecs()); + app.getDeployment().map(DeploymentSpec::fromXml) + .ifPresent(deploymentSpec -> { + addIdentityProvider(cluster, + context.getDeployState().getProperties().configServerSpecs(), + context.getDeployState().getProperties().loadBalancerName(), + context.getDeployState().zone(), + deploymentSpec); + + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getRotations(), deploymentSpec); + }); //TODO: overview handler, see DomQrserverClusterBuilder } + private void addRotationProperties(ContainerCluster cluster, Zone zone, Set<Rotation> rotations, DeploymentSpec spec) { + cluster.getContainers().forEach(container -> { + setRotations(container, rotations, spec.globalServiceId(), cluster.getName()); + container.setProp("activeRotation", Boolean.toString(zoneHasActiveRotation(zone, spec))); + }); + } + + private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { + return spec.zones().stream() + .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && + declaredZone.active()); + } + + private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) { + + if ( ! rotations.isEmpty() && globalServiceId.isPresent()) { + if (containerClusterName.equals(globalServiceId.get())) { + container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(","))); + } + } + } + private void addRoutingAliases(ContainerCluster cluster, Element spec, Environment environment) { if (environment != Environment.prod) return; @@ -691,26 +731,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addIdentity(Element element, ContainerCluster cluster, List<ConfigServerSpec> configServerSpecs) { - Element identityElement = XML.getChild(element, "identity"); - if(identityElement != null) { - String domain = XML.getValue(XML.getChild(identityElement, "domain")); - String service = XML.getValue(XML.getChild(identityElement, "service")); - - // TODO: Inject the load balancer address. For now only add first configserver - String cfgHostName = configServerSpecs.stream().findFirst().map(ConfigServerSpec::getHostName) - .orElse(""); // How to test this? - - Identity identity = new Identity(domain.trim(), service.trim(), cfgHostName); - cluster.addComponent(identity); + private void addIdentityProvider(ContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, Zone zone, DeploymentSpec spec) { + spec.athenzDomain().ifPresent(domain -> { + AthenzService service = spec.athenzService(zone.environment(), zone.region()) + .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration")); + IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs)); + cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { - container.setProp("identity.domain", domain); - container.setProp("identity.service", service); + container.setProp("identity.domain", domain.value()); + container.setProp("identity.service", service.value()); }); - } + }); + } + + private HostName getLoadBalancerName(HostName loadbalancerName, List<ConfigServerSpec> configServerSpecs) { + // Set lbaddress, or use first hostname if not specified. + // TODO: Remove this method and use the loadbalancerName directly + return Optional.ofNullable(loadbalancerName) + .orElseGet( + () -> HostName.from(configServerSpecs.stream() + .findFirst() + .map(ConfigServerSpec::getHostName) + .orElse("unknown") // Currently unable to test this, hence the unknown + )); } + /** * Disallow renderers named "DefaultRenderer" or "JsonRenderer" */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 2bfa8fa93a0..81aca977400 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.document.select.DocumentSelector; @@ -18,11 +20,15 @@ import java.util.logging.Logger; /** * Generates distributor-specific configuration. */ -public class DistributorCluster extends AbstractConfigProducer<Distributor> - implements StorDistributormanagerConfig.Producer, StorServerConfig.Producer, MetricsmanagerConfig.Producer { +public class DistributorCluster extends AbstractConfigProducer<Distributor> implements + StorDistributormanagerConfig.Producer, + StorServerConfig.Producer, + MetricsmanagerConfig.Producer, + BucketspacesConfig.Producer { public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString()); + private static class GcOptions { public final int interval; public final String selection; @@ -145,6 +151,20 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> builder.is_distributor(true); } + private static final String DEFAULT_BUCKET_SPACE = "default"; + private static final String GLOBAL_BUCKET_SPACE = "global"; + + @Override + public void getConfig(BucketspacesConfig.Builder builder) { + for (NewDocumentType docType : parent.getDocumentDefinitions().values()) { + BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); + docTypeBuilder.name(docType.getName()); + String bucketSpace = (parent.isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + docTypeBuilder.bucketspace(bucketSpace); + builder.documenttype(docTypeBuilder); + } + } + public String getClusterName() { return parent.getName(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index c7755d3de5a..7889b857fff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -524,6 +524,10 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri */ public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; } + public boolean isGloballyDistributed(NewDocumentType docType) { + return globallyDistributedDocuments.contains(docType); + } + public final ContentSearchCluster getSearch() { return search; } public Redundancy redundancy() { return redundancy; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 8860f5c2249..f6cc9203d00 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -110,11 +110,5 @@ public class FileDistributor { public void reloadDeployFileDistributor(FileDistribution dbHandler) { dbHandler.reloadDeployFileDistributor(); } - - private Set<String> union(Set<String> hosts, String... additionalHosts) { - Set<String> result = new HashSet<>(hosts); - result.addAll(asList(additionalHosts)); - return result; - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java index 6c2da51f3e3..dd9e057e2fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java @@ -54,8 +54,10 @@ public class FileDistributorService extends AbstractService implements @Override public String getStartupCommand() { - return "exec $ROOT/sbin/vespa-filedistributor" - + " --configid " + getConfigId(); + // If disabled config proxy should act as file distributor, so don't start this service + return (fileDistributionOptions.disabled()) + ? null + : "exec $ROOT/sbin/vespa-filedistributor" + " --configid " + getConfigId(); } @Override @@ -88,7 +90,9 @@ public class FileDistributorService extends AbstractService implements @Override public void getConfig(FiledistributorrpcConfig.Builder builder) { - builder.connectionspec("tcp/" + getHostName() + ":" + getRelativePort(0)); + // If disabled config proxy should act as file distributor, so use config proxy port + int port = (fileDistributionOptions.disabled()) ? 19090 : getRelativePort(0); + builder.connectionspec("tcp/" + getHostName() + ":" + port); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 7e0cb57aaff..8d7d289b99f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -192,14 +192,15 @@ public abstract class IndexedSearchCluster extends SearchCluster */ public void addTldsWithSameIdsAsContainers(AbstractConfigProducer tldParent, ContainerCluster containerCluster) { for (Container container : containerCluster.getContainers()) { - final String containerSubId = container.getSubId(); - if (!containerSubId.contains(".")) { + String containerSubId = container.getSubId(); + if ( ! containerSubId.contains(".")) { throw new RuntimeException("Expected container sub id to be of the form string.number"); } int containerIndex = Integer.parseInt(containerSubId.split("\\.")[1]); - final String containerClusterName = containerCluster.getName(); + String containerClusterName = containerCluster.getName(); log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() + - ", container cluster " + containerClusterName + " (container id " + containerSubId + ") on host " + container.getHostResource().getHostName()); + ", container cluster " + containerClusterName + " (container id " + containerSubId + + ") on host " + container.getHostResource().getHostName()); rootDispatch.addDispatcher(createTld(tldParent, container.getHostResource(), containerClusterName, containerIndex)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java index ba3d187bf8d..a0ec1bb9d22 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.search; /** - * Represents the row id and partition id of a search interface node. + * Represents the group and partition id of a search interface node. * * @author geirst */ diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index cdf26807bcb..b7ec252c4d7 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -5,7 +5,6 @@ AdminV2 = element admin { attribute version { "2.0" } & element adminserver { service.attlist } & - element minSlobroksPerCluster { xsd:positiveInteger }? & # TODO: Remove when no applications use this GenericConfig* & LogServer? & (ConfigServer | ConfigServers)? & @@ -20,7 +19,6 @@ AdminV2 = AdminV3 = element admin { attribute version { "3.0" } & - element minSlobroksPerCluster { xsd:positiveInteger }? & # TODO: Remove when no applications use this GenericConfig* & Nodes } @@ -28,7 +26,6 @@ AdminV3 = AdminV4 = element admin { attribute version { "4.0" } & - element minSlobroksPerCluster { xsd:positiveInteger }? & # TODO: Remove when no applications use this AdminV4Slobroks? & AdminV4LogServers? & GenericConfig* & @@ -83,7 +80,8 @@ LogServer = element logserver { FileDistribution = element filedistribution { element uploadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & - element downloadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? + element downloadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & + element disabled { xsd:boolean }? # Nov. 2017: Temporary, should not be documented } Metrics = element metrics { diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index af9b89b8553..6a90bef7bb2 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -7,8 +7,7 @@ ContainerCluster = element container | jdisc { ContainerServices & DocumentBinding* & Aliases? & - NodesOfContainerCluster? & - Identity? + NodesOfContainerCluster? } ContainerServices = @@ -62,6 +61,7 @@ Filtering = element filtering { HttpServer = element server { attribute port { xsd:nonNegativeInteger } & ComponentId & + element ssl-keystore-configurator { BundleSpec }? & # FOR INTERNAL USE ONLY - SUBJECT TO CHANGE GenericConfig* } @@ -225,8 +225,3 @@ DocumentBinding = element document { attribute class { xsd:NCName } & attribute bundle { xsd:NCName } } - -Identity = element identity { - element domain { xsd:NCName } & - element service { xsd:NCName } -} diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 90bff8e31b3..2ecfa781876 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -4,6 +4,8 @@ start = element deployment { attribute version { "1.0" } & + attribute athenz-domain { xsd:string }? & + attribute athenz-service { xsd:string }? & Upgrade? & BlockChange* & Test? & @@ -31,15 +33,18 @@ BlockUpgrade = element block-upgrade { # Legacy name - remove on Vespa 7 } Test = element test { - text + attribute athenz-service { xsd:string }? & + text } Staging = element staging { - text + attribute athenz-service { xsd:string }? & + text } Prod = element prod { attribute global-service-id { text }? & + attribute athenz-service { xsd:string }? & Region* & Delay* & Parallel* diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 83b5003579e..58f83d1e4e6 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -2,15 +2,20 @@ package com.yahoo.config.model; import com.yahoo.component.Version; -import com.yahoo.config.model.api.*; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; @@ -95,6 +100,11 @@ public class MockModelContext implements ModelContext { } @Override + public HostName loadBalancerName() { + return null; + } + + @Override public boolean hostedVespa() {return false; } @Override diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java new file mode 100644 index 00000000000..12bdd8d2b5c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -0,0 +1,206 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +public class TensorTransformTestCase extends SearchDefinitionTestCase { + + @Test + public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { + assertContainsExpression("max(1.0,2.0)", "max(1.0,2.0)"); + assertContainsExpression("min(attribute(double_field),x)", "min(attribute(double_field),x)"); + assertContainsExpression("max(attribute(double_field),attribute(double_array_field))", "max(attribute(double_field),attribute(double_array_field))"); + assertContainsExpression("min(attribute(tensor_field_1),attribute(double_field))", "min(attribute(tensor_field_1),attribute(double_field))"); + assertContainsExpression("max(attribute(tensor_field_1),attribute(tensor_field_2))", "max(attribute(tensor_field_1),attribute(tensor_field_2))"); + assertContainsExpression("min(test_constant_tensor,1.0)", "min(constant(test_constant_tensor),1.0)"); + assertContainsExpression("max(base_constant_tensor,1.0)", "max(constant(base_constant_tensor),1.0)"); + assertContainsExpression("min(constant(file_constant_tensor),1.0)", "min(constant(file_constant_tensor),1.0)"); + assertContainsExpression("max(query(q),1.0)", "max(query(q),1.0)"); + assertContainsExpression("max(query(n),1.0)", "max(query(n),1.0)"); + } + + @Test + public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { + assertContainsExpression("max(attribute(tensor_field_1),x)", "reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("1 + max(attribute(tensor_field_1),x)", "1+reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)", "if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)"); + assertContainsExpression("max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(max(attribute(tensor_field_1),x),x)", "max(reduce(attribute(tensor_field_1),max,x),x)"); // will result in deploy error. + assertContainsExpression("max(max(attribute(tensor_field_2),x),y)", "reduce(reduce(attribute(tensor_field_2),max,x),max,y)"); + } + + @Test + public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { + assertContainsExpression("max(test_constant_tensor,x)", "reduce(constant(test_constant_tensor),max,x)"); + assertContainsExpression("max(base_constant_tensor,x)", "reduce(constant(base_constant_tensor),max,x)"); + assertContainsExpression("min(constant(file_constant_tensor),x)", "reduce(constant(file_constant_tensor),min,x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { + assertContainsExpression("min(attribute(double_field) + attribute(tensor_field_1),x)", "reduce(attribute(double_field)+attribute(tensor_field_1),min,x)"); + assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); + assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field)),double_array_field)", "reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field),x),x)", "reduce(tensorFromLabels(attribute(double_array_field),x),max,x)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)", "reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field),x),x)", "reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { + assertContainsExpression("max(query(q),x)", "reduce(query(q),max,x)"); + assertContainsExpression("max(query(n),x)", "max(query(n),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensoresReturnedFromMacrosAreReplaced() throws ParseException { + assertContainsExpression("max(returns_tensor,x)", "reduce(rankingExpression(returns_tensor),max,x)"); + assertContainsExpression("max(wraps_returns_tensor,x)", "reduce(rankingExpression(wraps_returns_tensor),max,x)"); + assertContainsExpression("max(tensor_inheriting,x)", "reduce(rankingExpression(tensor_inheriting),max,x)"); + assertContainsExpression("max(returns_tensor_with_arg(attribute(tensor_field_1)),x)", "reduce(rankingExpression(returns_tensor_with_arg@),max,x)"); + } + + + private void assertContainsExpression(String expr, String transformedExpression) throws ParseException { + assertTrue("Expected expression '" + transformedExpression + "' not found", + containsExpression(expr, transformedExpression)); + } + + private boolean containsExpression(String expr, String transformedExpression) throws ParseException { + for (Pair<String, String> rankPropertyExpression : buildSearch(expr)) { + String rankProperty = rankPropertyExpression.getFirst(); + if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) { + String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); + return rankExpression.equals(transformedExpression); + } + } + return false; + } + + private List<Pair<String, String>> buildSearch(String expression) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field double_field type double { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field double_array_field type array<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field weightedset_field type weightedset<double> { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_1 type tensor(x{}) { \n" + + " indexing: summary | attribute \n" + + " attribute: tensor(x{}) \n" + + " }\n" + + " field tensor_field_2 type tensor(x[3],y[3]) { \n" + + " indexing: summary | attribute \n" + + " attribute: tensor(x[3],y[3]) \n" + + " }\n" + + " }\n" + + " constant file_constant_tensor {\n" + + " file: constants/tensor.json\n" + + " type: tensor(x{})\n" + + " }\n" + + " rank-profile base {\n" + + " constants {\n" + + " base_constant_tensor {\n" + + " value: { {x:0}:0 }\n" + + " }\n" + + " }\n" + + " macro base_tensor() {\n" + + " expression: constant(base_constant_tensor)\n" + + " }\n" + + " }\n" + + " rank-profile test inherits base {\n" + + " constants {\n" + + " test_constant_tensor {\n" + + " value: { {x:0}:1 }\n" + + " }\n" + + " }\n" + + " macro returns_tensor_with_arg(arg1) {\n" + + " expression: 2.0 * arg1\n" + + " }\n" + + " macro wraps_returns_tensor() {\n" + + " expression: returns_tensor\n" + + " }\n" + + " macro returns_tensor() {\n" + + " expression: attribute(tensor_field_2)\n" + + " }\n" + + " macro tensor_inheriting() {\n" + + " expression: base_tensor\n" + + " }\n" + + " first-phase {\n" + + " expression: " + expression + "\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); + Search s = builder.getSearch(); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); + for (Object o : testRankProperties) + System.out.println(o); + return testRankProperties; + } + + private static QueryProfiles setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(q)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(n)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + return new QueryProfiles(registry); + } + + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + if ( ! areInHash) + b.append(current); + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index cef94c97a2c..9a507827d17 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -18,6 +18,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.ProvisionLogger; @@ -193,6 +194,11 @@ public class VespaModelFactoryTest { public List<ConfigServerSpec> configServerSpecs() { return Collections.emptyList(); } + + @Override + public HostName loadBalancerName() { + return null; + } }; } }; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index b5375ccbce4..946624a1cdb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -275,4 +275,36 @@ public class AdminTestCase { // 1 logforwarder on each host assertTrue(configIds.toString(), configIds.contains("admin/logforwarder.0")); } + + @Test + public void disableFiledistributorService() throws Exception { + String hosts = "<hosts>" + + " <host name=\"localhost\">" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + String services = "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " <filedistribution>" + + " <disabled>true</disabled>" + + " </filedistribution>" + + " </admin>" + + "</services>"; + + VespaModel vespaModel = new VespaModelCreatorWithMockPkg(hosts, services).create(); + String localhost = HostName.getLocalhost(); + String localhostConfigId = "hosts/" + localhost; + + // Verify services in the sentinel config + SentinelConfig.Builder b = new SentinelConfig.Builder(); + vespaModel.getConfig(b, localhostConfigId); + SentinelConfig sentinelConfig = new SentinelConfig(b); + assertThat(sentinelConfig.service().size(), is(3)); + assertThat(sentinelConfig.service(0).name(), is("logserver")); + assertThat(sentinelConfig.service(1).name(), is("slobrok")); + assertThat(sentinelConfig.service(2).name(), is("logd")); + // No filedistributor service + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index af846f10ffe..d9c151480fe 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -18,15 +18,14 @@ import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerClusterVerifier; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import org.junit.Test; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.Optional; @@ -266,48 +265,6 @@ public class ContainerClusterTest { assertEquals(0, cluster.getAllComponents().stream().map(c -> c.getClassId().getName()).filter(c -> c.equals("com.yahoo.jdisc.http.filter.security.RoutingConfigProvider")).count()); } - @Test - public void setsRotationActiveAccordingToDeploymentSpec() { - String deploymentSpec = "<deployment>\n" + - " <prod> \n" + - " <region active='true'>us-north-1</region>\n" + - " <parallel>\n" + - " <region active='false'>us-north-2</region>\n" + - " <region active='true'>us-north-3</region>\n" + - " </parallel>\n" + - " <region active='false'>us-north-4</region>\n" + - " </prod>\n" + - "</deployment>"; - for (String region : Arrays.asList("us-north-1", "us-north-3")) { - Container container = containerIn(region, deploymentSpec); - assertEquals("Region " + region + " is active", "true", - container.getServicePropertyString("activeRotation")); - } - for (String region : Arrays.asList("us-north-2", "us-north-4")) { - Container container = containerIn(region, deploymentSpec); - assertEquals("Region " + region + " is inactive", "false", - container.getServicePropertyString("activeRotation")); - } - Container container = containerIn("unknown", deploymentSpec); - assertEquals("Unknown region is inactive", "false", - container.getServicePropertyString("activeRotation")); - } - - private static Container containerIn(String regionName, String deploymentSpec) { - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() - .withDeploymentSpec(deploymentSpec) - .build(); - DeployState state = new DeployState.Builder() - .applicationPackage(applicationPackage) - .properties(new DeployProperties.Builder().hostedVespa(true).zone( - new Zone(Environment.prod, RegionName.from(regionName))).build() - ) - .build(); - MockRoot root = new MockRoot("foo", state); - ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); - addContainer(cluster, "c1", "c1.domain"); - return cluster.getContainers().get(0); - } private static void addContainer(ContainerCluster cluster, String name, String hostName) { Container container = new Container(cluster, name, 0); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 0e6bce73867..d09211aea45 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -26,7 +26,6 @@ import com.yahoo.container.servlet.ServletConfigConfig; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.prelude.cluster.QrMonitorConfig; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; @@ -46,6 +45,7 @@ import java.util.Map; import java.util.logging.Level; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java index df118b0e349..d3ad2ccc721 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/IdentityBuilderTest.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.vespa.model.container.Identity; +import com.yahoo.vespa.model.container.IdentityProvider; import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -17,17 +20,23 @@ import static org.junit.Assert.assertEquals; */ public class IdentityBuilderTest extends ContainerModelBuilderTestBase { @Test - public void identity_config_produced() throws IOException, SAXException { + public void identity_config_produced_from_deployment_spec() throws IOException, SAXException { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", - " <identity>", - " <domain>domain</domain>", - " <service>service</service>", - " </identity>", - "</jdisc>"); + "<jdisc id='default' version='1.0'><search /></jdisc>"); + String deploymentXml = "<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + + " <test/>\n" + + " <prod>\n" + + " <region active='true'>default</region>\n" + + " </prod>\n" + + "</deployment>\n"; - createModel(root, clusterElem); - IdentityConfig identityConfig = root.getConfig(IdentityConfig.class, "default/component/" + Identity.CLASS); + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withDeploymentSpec(deploymentXml) + .build(); + + createModel(root, DeployState.createTestState(applicationPackage), clusterElem); + + IdentityConfig identityConfig = root.getConfig(IdentityConfig.class, "default/component/" + IdentityProvider.CLASS); assertEquals("domain", identityConfig.domain()); assertEquals("service", identityConfig.service()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 4621b5ebe50..1e24b055095 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -3,17 +3,24 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.jdisc.FilterBindingsProvider; import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.JettyHttpServer; import org.junit.Test; import org.w3c.dom.Element; +import org.xml.sax.SAXException; +import java.io.IOException; import java.util.List; import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; @@ -182,6 +189,42 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas is(not(nullValue()))); } + @Test + public void ssl_keystore_configurator_can_be_overriden() throws IOException, SAXException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + " <http>", + " <server port='9000' id='foo'>", + " <ssl-keystore-configurator class='com.yahoo.MySslKeyStoreConfigurator' bundle='mybundle'/>", + " </server>", + " <server port='9001' id='bar'/>", + " </http>", + nodesXml, + "</jdisc>"); + createModel(root, clusterElem); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); + List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); + + { + ConnectorFactory firstConnector = connectorFactories.get(0); + assertThat(firstConnector.getInjectedComponentIds(), hasItem("ssl-keystore-configurator@foo")); + assertThat(firstConnector.getInjectedComponentIds().size(), equalTo(1)); + SimpleComponent sslKeystoreConfigurator = firstConnector.getChildrenByTypeRecursive(SimpleComponent.class).get(0); + BundleInstantiationSpecification spec = sslKeystoreConfigurator.model.bundleInstantiationSpec; + assertThat(spec.classId.toString(), is("com.yahoo.MySslKeyStoreConfigurator")); + assertThat(spec.bundle.toString(), is("mybundle")); + } + { + ConnectorFactory secondFactory = connectorFactories.get(1); + assertThat(secondFactory.getInjectedComponentIds(), hasItem("ssl-keystore-configurator@bar")); + assertThat(secondFactory.getInjectedComponentIds().size(), equalTo(1)); + SimpleComponent sslKeystoreConfigurator = secondFactory.getChildrenByTypeRecursive(SimpleComponent.class).get(0); + BundleInstantiationSpecification spec = sslKeystoreConfigurator.model.bundleInstantiationSpec; + assertThat(spec.classId.toString(), is(DefaultSslKeyStoreConfigurator.class.getName())); + assertThat(spec.bundle.toString(), is("jdisc_http_service")); + } + } + private void assertJettyServerInConfig() { ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<JettyHttpServer> jettyServers = cluster.getChildrenByTypeRecursive(JettyHttpServer.class); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java new file mode 100644 index 00000000000..a2f32694340 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class RoutingBuilderTest extends ContainerModelBuilderTestBase { + + @Test + public void setsRotationActiveAccordingToDeploymentSpec() throws IOException, SAXException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'><search /></jdisc>"); + + String deploymentSpec = "<deployment>\n" + + " <prod> \n" + + " <region active='true'>us-north-1</region>\n" + + " <parallel>\n" + + " <region active='false'>us-north-2</region>\n" + + " <region active='true'>us-north-3</region>\n" + + " </parallel>\n" + + " <region active='false'>us-north-4</region>\n" + + " </prod>\n" + + "</deployment>"; + + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withDeploymentSpec(deploymentSpec) + .build(); + //root = new MockRoot("root", applicationPackage); + for (String region : Arrays.asList("us-north-1", "us-north-3")) { + Container container = getContainer(applicationPackage, region, clusterElem); + + assertEquals("Region " + region + " is active", "true", + container.getServicePropertyString("activeRotation")); + } + for (String region : Arrays.asList("us-north-2", "us-north-4")) { + Container container = getContainer(applicationPackage, region, clusterElem); + + assertEquals("Region " + region + " is inactive", "false", + container.getServicePropertyString("activeRotation")); + } + Container container = getContainer(applicationPackage, "unknown", clusterElem); + assertEquals("Unknown region is inactive", "false", + container.getServicePropertyString("activeRotation")); + } + + + private Container getContainer(ApplicationPackage applicationPackage, String region, Element clusterElem) throws IOException, SAXException { + DeployState deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from(region))) + .build(); + + root = new MockRoot("root", deployState); + createModel(root, deployState, clusterElem); + ContainerCluster cluster = getContainerCluster("default"); + return cluster.getContainers().get(0); + + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index 624d3e8ded8..d4e804d3f95 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; @@ -305,22 +306,28 @@ public class DistributorTest { private static class DocDef { public final String type; public final String mode; + public final boolean global; - private DocDef(String type, String mode) { + private DocDef(String type, String mode, boolean global) { this.type = type; this.mode = mode; + this.global = global; } public static DocDef storeOnly(String type) { - return new DocDef(type, "store-only"); + return new DocDef(type, "store-only", false); } public static DocDef index(String type) { - return new DocDef(type, "index"); + return new DocDef(type, "index", false); + } + + public static DocDef indexGlobal(String type) { + return new DocDef(type, "index", true); } public static DocDef streaming(String type) { - return new DocDef(type, "streaming"); + return new DocDef(type, "streaming", false); } } @@ -328,7 +335,8 @@ public class DistributorTest { return "<content id='storage'>\n" + " <documents>\n" + Arrays.stream(defs) - .map(def -> String.format(" <document type='%s' mode='%s'/>", def.type, def.mode)) + .map(def -> String.format(" <document type='%s' mode='%s' global='%s'/>", + def.type, def.mode, (def.global ? "true" : "false"))) .collect(Collectors.joining("\n")) + "\n </documents>\n" + "</content>"; @@ -371,4 +379,24 @@ public class DistributorTest { generateXmlForDocDefs(DocDef.streaming("music"))); assertThat(config.disable_bucket_activation(), is(true)); } + + private BucketspacesConfig clusterXmlToBucketspacesConfig(String xml) { + BucketspacesConfig.Builder builder = new BucketspacesConfig.Builder(); + parse(xml).getConfig(builder); + return new BucketspacesConfig(builder); + } + + private void assertDocumentType(String expName, String expBucketSpace, BucketspacesConfig.Documenttype docType) { + assertEquals(expName, docType.name()); + assertEquals(expBucketSpace, docType.bucketspace()); + } + + @Test + public void bucket_spaces_config_is_produced_for_distributor_cluster() { + BucketspacesConfig config = clusterXmlToBucketspacesConfig( + generateXmlForDocDefs(DocDef.index("music"), DocDef.indexGlobal("movies"))); + assertEquals(2, config.documenttype().size()); + assertDocumentType("movies", "global", config.documenttype(0)); + assertDocumentType("music", "default", config.documenttype(1)); + } } diff --git a/config-model/src/test/schema-test-files/deployment.xml b/config-model/src/test/schema-test-files/deployment.xml index f469d22b6f0..f9a62eb648f 100644 --- a/config-model/src/test/schema-test-files/deployment.xml +++ b/config-model/src/test/schema-test-files/deployment.xml @@ -1,12 +1,12 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<deployment version='1.0'> +<deployment version='1.0' athenz-domain='vespa' athenz-service='service'> <upgrade policy='canary'/> <test/> <staging/> <block-change revision='true' version='false' days="mon,tue" hours="14,15"/> <block-change days="mon,tue" hours="14,15" time-zone="CET"/> <block-upgrade days="wed" hours="16" time-zone="CET"/><!-- Tests legacy name. Remove in Vespa 7 --> - <prod global-service-id='qrs'> + <prod global-service-id='qrs' athenz-service='other-service'> <region active='true'>us-west-1</region> <delay hours='3'/> <region active='true'>us-central-1</region> diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml index aee208088b7..90894aa253b 100644 --- a/config-model/src/test/schema-test-files/services-hosted.xml +++ b/config-model/src/test/schema-test-files/services-hosted.xml @@ -4,7 +4,6 @@ <admin version="3.0"> <nodes count="3" flavor="small"/> - <minSlobroksPerCluster>1</minSlobroksPerCluster> </admin> <jdisc id="container1" version="1.0"> diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 380ce7f5a3d..a02346193cc 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -7,7 +7,6 @@ </config> <admin version="2.0"> - <minSlobroksPerCluster>1</minSlobroksPerCluster> <adminserver hostalias="adminserver" /> <logserver hostalias="logserver" /> <slobroks> @@ -36,10 +35,6 @@ </config> <jdisc id='qrsCluster_1' version='1.0'> - <identity> - <domain>mydomain</domain> - <service>myservice</service> - </identity> <rest-api path="jersey1"> <components bundle="my-bundle" /> <components bundle="other-bundle"> @@ -111,7 +106,9 @@ </request-chain> </filtering> - <server port="4080" id="myServer" /> + <server port="4080" id="myServer"> + <ssl-keystore-configurator class="com.yahoo.MySslKeyStoreConfigurator" bundle="mybundle" /> + </server> <server port="4081" id="anotherServer"> <config name="container.jdisc.config.http-server"> <maxChunkSize>9999</maxChunkSize> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java new file mode 100644 index 00000000000..193a02ccf26 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +/** + * @author mortent + */ +public class AthenzDomain { + + private final String name; + + private AthenzDomain(String name) { + this.name = name; + } + + public static AthenzDomain from(String value) { + return new AthenzDomain(value); + } + + public String value() { return name; } + + @Override + public String toString() { + return "AthenzDomain{" + + "name='" + name + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AthenzDomain that = (AthenzDomain) o; + + return name != null ? name.equals(that.name) : that.name == null; + } + + @Override + public int hashCode() { + return name != null ? name.hashCode() : 0; + } +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzService.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzService.java new file mode 100644 index 00000000000..54f2889d95c --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzService.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +/** + * @author mortent + */ +public class AthenzService { + + private final String name; + + private AthenzService(String name) { + this.name = name; + } + + public String value() { return name; } + + public static AthenzService from(String value) { + return new AthenzService(value); + } + + @Override + public String toString() { + return "AthenzService{" + + "name='" + name + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AthenzService that = (AthenzService) o; + + return name != null ? name.equals(that.name) : that.name == null; + } + + @Override + public int hashCode() { + return name != null ? name.hashCode() : 0; + } +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index fc3e46a46d6..b0c04cea1b0 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -76,6 +76,9 @@ public class Flavor { } public Type getType() { return type; } + + /** Convenience, returns getType() == Type.DOCKER_CONTAINER */ + public boolean isDocker() { return type == Type.DOCKER_CONTAINER; } /** The free capacity we would like to preserve for this flavor */ public int getIdealHeadroom() { @@ -131,6 +134,14 @@ public class Flavor { public void freeze() { replacesFlavors = ImmutableList.copyOf(replacesFlavors); } + + /** Returns whether this flavor has at least as much as each hardware resource as the given flavor */ + public boolean isLargerThan(Flavor other) { + return this.minCpuCores >= other.minCpuCores && + this.minDiskAvailableGb >= other.minDiskAvailableGb && + this.minMainMemoryAvailableGb >= other.minMainMemoryAvailableGb && + this.fastDisk || ! other.fastDisk; + } @Override public int hashCode() { return name.hashCode(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java new file mode 100644 index 00000000000..510122c2342 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import java.util.Objects; + +/** + * Represents a host name + * + * @author mortent + */ +public class HostName implements Comparable<HostName> { + + private final String name; + + private HostName(String name) { + this.name = name; + } + + public String value() { return name; } + + /** + * Create a {@link HostName} with a given name. + * + * @param name Name + * @return instance of {@link HostName}. + */ + public static HostName from(String name) { + return new HostName(name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof HostName)) return false; + return Objects.equals(((HostName)obj).value(), value()); + } + + @Override + public String toString() { + return name; + } + + @Override + public int compareTo(HostName that) { + return this.name.compareTo(that.name); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 36be3707206..df2c4f85db9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -8,7 +8,10 @@ package com.yahoo.config.provision; */ public enum SystemName { - /** System for continuous deployment where a pre-test of hosted Vespa combined with a verified Vespa version */ + /** Local development system */ + dev, + + /** Continuous deployment system */ cd, /** Production system */ @@ -20,6 +23,7 @@ public enum SystemName { public static SystemName from(String value) { switch (value) { + case "dev": return dev; case "cd": return cd; case "main": return main; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index a2dcfd3fd07..57affc2f104 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -25,7 +25,7 @@ flavor[].cost int default=0 # for some historical reason. These nodes are assigned to applications by exact match and ignoring cost. flavor[].stock bool default=true -# The type of node (e.g. bare metal, docker..). +# The type of node: BARE_METAL, VIRTUAL_MACHINE or DOCKER_CONTAINER flavor[].environment string default="undefined" # The minimum number of CPU cores available. diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index 8a7b6f253e2..0aadb1bbb12 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -57,10 +57,11 @@ <artifactId>hamcrest-core</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>filedistribution</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> <plugins> diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 1fced0b1e3d..fe5976e9c1f 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.config.FileReference; import com.yahoo.jrt.*; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.*; @@ -10,12 +9,10 @@ import com.yahoo.vespa.config.protocol.JRTConfigRequestFactory; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; -import java.io.File; -import java.lang.*; -import java.util.*; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Set; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** @@ -30,14 +27,14 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer private static final int TRACELEVEL = 6; private final Spec spec; - private final Supervisor supervisor = new Supervisor(new Transport()); + private final Supervisor supervisor; private final ProxyServer proxyServer; - ConfigProxyRpcServer(ProxyServer proxyServer, Spec spec) { + ConfigProxyRpcServer(ProxyServer proxyServer, Supervisor supervisor, Spec spec) { this.proxyServer = proxyServer; this.spec = spec; + this.supervisor = supervisor; declareConfigMethods(); - declareFileDistributionMethods(); } public void run() { @@ -105,30 +102,6 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer .returnDesc(0, "ret", "Empty string or error message")); } - private void declareFileDistributionMethods() { - // Legacy method, needs to be the same name as used in filedistributor - supervisor.addMethod(new Method("waitFor", "s", "s", - this, "getFile") - .methodDesc("wait for file reference") - .paramDesc(0, "file reference", "file reference") - .returnDesc(0, "path", "path to file")); - supervisor.addMethod(new Method("filedistribution.getFile", "s", "s", - this, "getFile") - .methodDesc("wait for file reference") - .paramDesc(0, "file reference", "file reference") - .returnDesc(0, "path", "path to file")); - supervisor.addMethod(new Method("filedistribution.getActiveFileReferencesStatus", "", "SD", - this, "getActiveFileReferencesStatus") - .methodDesc("download status for file references") - .returnDesc(0, "file references", "array of file references") - .returnDesc(1, "download status", "percentage downloaded of each file reference in above array")); - supervisor.addMethod(new Method("filedistribution.setFileReferencesToDownload", "S", "i", - this, "setFileReferencesToDownload") - .methodDesc("set which file references to download") - .paramDesc(0, "file references", "file reference to download") - .returnDesc(0, "ret", "0 if success, 1 otherwise")); - } - //---------------- RPC methods ------------------------------------ /** @@ -235,50 +208,6 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer req.returnValues().add(new StringValue(memoryCache.dumpCacheToDisk(req.parameters().get(0).asString(), memoryCache))); } - @SuppressWarnings({"UnusedDeclaration"}) - public final void getFile(Request req) { - // TODO: Detach to avoid holding transport thread - FileReference fileReference = new FileReference(req.parameters().get(0).asString()); - String pathToFile = proxyServer.fileDownloader() - .getFile(fileReference) - .orElseGet(() -> new File("")) - .getAbsolutePath(); - - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' available at " + pathToFile); - req.returnValues().add(new StringValue(pathToFile)); - } - - @SuppressWarnings({"UnusedDeclaration"}) - public final void getActiveFileReferencesStatus(Request req) { - Map<FileReference, Double> downloadStatus = proxyServer.fileDownloader().downloadStatus(); - - String[] fileRefArray = new String[downloadStatus.keySet().size()]; - fileRefArray = downloadStatus.keySet().stream() - .map(FileReference::value) - .collect(Collectors.toList()) - .toArray(fileRefArray); - - double[] downloadStatusArray = new double[downloadStatus.values().size()]; - int i = 0; - for (Double d : downloadStatus.values()) { - downloadStatusArray[i++] = d; - } - - req.returnValues().add(new StringArray(fileRefArray)); - req.returnValues().add(new DoubleArray(downloadStatusArray)); - } - - @SuppressWarnings({"UnusedDeclaration"}) - public final void setFileReferencesToDownload(Request req) { - String[] fileReferenceStrings = req.parameters().get(0).asStringArray(); - List<FileReference> fileReferences = Stream.of(fileReferenceStrings) - .map(FileReference::new) - .collect(Collectors.toList()); - proxyServer.fileDownloader().queueForDownload(fileReferences); - - req.returnValues().add(new Int32Value(0)); - } - //---------------------------------------------------- private boolean isProtocolVersionSupported(JRTServerConfigRequest request) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 4ee77beb2d7..28bcca9db13 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -5,13 +5,16 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Spec; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.system.CatchSigTerm; import com.yahoo.vespa.config.*; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; -import com.yahoo.vespa.config.proxy.filedistribution.FileDownloader; +import com.yahoo.vespa.filedistribution.FileDistributionRpcServer; +import com.yahoo.vespa.filedistribution.FileDownloader; import java.util.List; import java.util.concurrent.Executors; @@ -40,6 +43,7 @@ public class ProxyServer implements Runnable { // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); + private final Supervisor supervisor = new Supervisor(new Transport()); private final ClientUpdater clientUpdater; private ScheduledFuture<?> delayedResponseScheduler; @@ -83,7 +87,8 @@ public class ProxyServer implements Runnable { this.rpcServer = createRpcServer(spec); clientUpdater = new ClientUpdater(rpcServer, statistics, delayedResponses); this.configClient = createClient(clientUpdater, delayedResponses, source, timingValues, memoryCache, configClient); - this.fileDownloader = new FileDownloader(source); + this.fileDownloader = new FileDownloader(new JRTConnectionPool(source)); + new FileDistributionRpcServer(supervisor, fileDownloader); } static ProxyServer createTestServer(ConfigSourceSet source) { @@ -162,7 +167,7 @@ public class ProxyServer implements Runnable { } private ConfigProxyRpcServer createRpcServer(Spec spec) { - return (spec == null) ? null : new ConfigProxyRpcServer(this, spec); // TODO: Try to avoid first argument being 'this' + return (spec == null) ? null : new ConfigProxyRpcServer(this, supervisor, spec); // TODO: Try to avoid first argument being 'this' } private RpcConfigSourceClient createRpcClient() { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java deleted file mode 100644 index 9074527e4e4..00000000000 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloader.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.proxy.filedistribution; - -import com.google.common.collect.ImmutableSet; -import com.yahoo.config.FileReference; -import com.yahoo.config.subscription.ConfigSourceSet; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.defaults.Defaults; - -import java.io.File; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.logging.Logger; - -/** - * Keeps track of files to download and download status - * - * @author hmusum - */ -public class FileDownloader { - private final static Logger log = Logger.getLogger(FileDownloader.class.getName()); - - - private final String filesDirectory; - private final ConfigSourceSet configSourceSet; - private final Duration timeout; - private final Map<FileReference, Double> downloadStatus = new HashMap<>(); - private final Set<FileReference> queuedForDownload = new LinkedHashSet<>(); - - public FileDownloader(ConfigSourceSet configSourceSet) { - this(configSourceSet, - Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"), - Duration.ofMinutes(15)); - } - - FileDownloader(ConfigSourceSet configSourceSet, String filesDirectory, Duration timeout) { - this.configSourceSet = configSourceSet; - this.filesDirectory = filesDirectory; - this.timeout = timeout; - } - - public Optional<File> getFile(FileReference fileReference) { - Objects.requireNonNull(fileReference, "file reference cannot be null"); - File directory = new File(filesDirectory, fileReference.value()); // directory with one file - - log.log(LogLevel.DEBUG, "Checking if there is a file in '" + directory.getAbsolutePath() + "' "); - Instant end = Instant.now().plus(timeout); - do { - File[] files = directory.listFiles(); - if (directory.exists() && directory.isDirectory() && files != null && files.length > 0) { - if (files.length != 1) { - throw new RuntimeException("More than one file in '" + fileReference.value() + - "', expected only one, unable to proceed"); - } - File file = files[0]; - if (!file.exists()) { - throw new RuntimeException("File with reference '" + fileReference.value() + - "' does not exist"); - } else if (!file.canRead()) { - throw new RuntimeException("File with reference '" + fileReference.value() + - "'exists, but unable to read it"); - } else { - downloadStatus.put(fileReference, 100.0); - return Optional.of(file); - } - } else { - queueForDownload(fileReference); - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } while (Instant.now().isBefore(end)); - - return Optional.empty(); - } - - public Map<FileReference, Double> downloadStatus() { - return downloadStatus; - } - - public void queueForDownload(List<FileReference> fileReferences) { - fileReferences.forEach(this::queueForDownload); - } - - private void queueForDownload(FileReference fileReference) { - log.log(LogLevel.INFO, "Queued '" + fileReference.value() + "' for download "); - queuedForDownload.add(fileReference); - downloadStatus.put(fileReference, 0.0); - } - - ImmutableSet<FileReference> queuedForDownload() { - return ImmutableSet.copyOf(queuedForDownload); - } - -}
\ No newline at end of file diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index c1e9826e29f..4a9d2acb4c5 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -5,6 +5,8 @@ import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.RawConfig; import org.junit.After; import org.junit.Before; @@ -28,7 +30,7 @@ public class ConfigProxyRpcServerTest { @Before public void setup() { proxyServer = ProxyServer.createTestServer(new ConfigSourceSet(address)); - rpcServer = new ConfigProxyRpcServer(proxyServer, null); + rpcServer = new ConfigProxyRpcServer(proxyServer, new Supervisor(new Transport()), null); } @After @@ -38,10 +40,9 @@ public class ConfigProxyRpcServerTest { @Test public void basic() { - ConfigSourceSet configSources = new ConfigSourceSet(); - ProxyServer proxy = ProxyServer.createTestServer(configSources); + ProxyServer proxy = ProxyServer.createTestServer(new MockConfigSource(new MockClientUpdater())); Spec spec = new Spec("localhost", 12345); - ConfigProxyRpcServer server = new ConfigProxyRpcServer(proxy, spec); + ConfigProxyRpcServer server = new ConfigProxyRpcServer(proxy, new Supervisor(new Transport()), spec); assertThat(server.getSpec(), is(spec)); } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java index dc9a3408510..2b26996fbdc 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java @@ -1,12 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.Set; /** * A simple class to be able to test config proxy without having an RPC config @@ -37,4 +39,9 @@ class MockConfigSource extends ConfigSourceSet { backing.clear(); } + @Override + public Set<String> getSources() { + return Collections.singleton("tcp/localhost:19070,tcp/localhost:19071,tcp/localhost:19072"); + } + } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index f82d9e90184..3cd0f1043cc 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -95,7 +95,6 @@ public class ProxyServerTest { */ @Test public void testModeSwitch() { - ConfigSourceSet source = new ConfigSourceSet(); // Need to use a ConfigSourceSet to test modes ProxyServer proxy = ProxyServer.createTestServer(source); assertTrue(proxy.getMode().isDefault()); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java deleted file mode 100644 index ea880e451b6..00000000000 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.yahoo.vespa.config.proxy.filedistribution; - -import com.yahoo.config.FileReference; -import com.yahoo.config.subscription.ConfigSourceSet; -import com.yahoo.io.IOUtils; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.time.Duration; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class FileDownloaderTest { - private static final ConfigSourceSet configSourceSet = new ConfigSourceSet(); - - @Test - public void download() throws IOException { - File downloadDir = Files.createTempDirectory("filedistribution").toFile(); - FileDownloader fileDownloader = new FileDownloader(configSourceSet, downloadDir.getAbsolutePath(), Duration.ofMillis(200)); - - // Write a file to download directory to simulate download going OK - String fileReferenceString = "somehash"; - String fileName = "foo.jar"; - File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReferenceString); - FileReference fileReference = writeFileReference(downloadDir, fileReferenceString, fileName); - - // Check that we get correct path and content when asking for file reference - Optional<File> pathToFile = fileDownloader.getFile(fileReference); - assertTrue(pathToFile.isPresent()); - String downloadedFile = new File(fileReferenceFullPath, fileName).getAbsolutePath(); - assertEquals(new File(fileReferenceFullPath, fileName).getAbsolutePath(), downloadedFile); - assertEquals("content", IOUtils.readFile(pathToFile.get())); - - // Verify download status - Map<FileReference, Double> downloadStatus = fileDownloader.downloadStatus(); - assertEquals(1, downloadStatus.size()); - assertDownloadStatus(Collections.singletonList(fileReference), downloadStatus.entrySet().iterator().next(), 100.0); - - // Non-existing file - assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(new FileReference("doesnotexist")).isPresent()); - } - - @Test - public void setFilesToDownload() throws IOException { - File downloadDir = Files.createTempDirectory("filedistribution").toFile(); - FileDownloader fileDownloader = new FileDownloader(configSourceSet, downloadDir.getAbsolutePath(), Duration.ofMillis(200)); - List<FileReference> fileReferences = Arrays.asList(new FileReference("foo"), new FileReference("bar")); - fileDownloader.queueForDownload(fileReferences); - - assertEquals(fileReferences, fileDownloader.queuedForDownload().asList()); - - // Verify download status - Map<FileReference, Double> downloadStatus = fileDownloader.downloadStatus(); - assertEquals(2, downloadStatus.size()); - - assertDownloadStatus(fileReferences, downloadStatus.entrySet().iterator().next(), 0.0); - assertDownloadStatus(fileReferences, downloadStatus.entrySet().iterator().next(), 0.0); - } - - private FileReference writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { - File file = new File(new File(dir, fileReferenceString), fileName); - IOUtils.writeFile(file, "content", false); - return new FileReference(fileReferenceString); - } - - private File fileReferenceFullPath(File dir, String fileReferenceString) { - return new File(dir, fileReferenceString); - } - - private void assertDownloadStatus(List<FileReference> fileReferences, Map.Entry<FileReference, Double> entry, double expectedDownloadStatus) { - assertTrue(fileReferences.contains(new FileReference(entry.getKey().value()))); - assertEquals(expectedDownloadStatus, entry.getValue(), 0.0001); - } -} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index 3509a960740..cb623437dba 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -3,6 +3,7 @@ package com.yahoo.config.subscription.impl; import com.yahoo.jrt.Request; import com.yahoo.jrt.RequestWaiter; +import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; @@ -53,6 +54,12 @@ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Co } @Override + public void invokeSync(Request request, double jrtTimeout) { + numberOfRequests++; + lastRequest = request; + } + + @Override public void setError(int errorCode) { numberOfFailovers++; } @@ -68,9 +75,7 @@ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Co } @Override - public void close() { - - } + public void close() {} @Override public void setError(Connection connection, int errorCode) { @@ -92,6 +97,11 @@ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Co return numSpecs; } + @Override + public Supervisor getSupervisor() { + return null; + } + public int getNumberOfRequests() { return numberOfRequests; } @@ -109,7 +119,6 @@ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Co } } - public interface ResponseHandler extends Runnable { RequestWaiter requestWaiter(); diff --git a/config/src/main/java/com/yahoo/vespa/config/Connection.java b/config/src/main/java/com/yahoo/vespa/config/Connection.java index 3d487198450..e39175a3a78 100644 --- a/config/src/main/java/com/yahoo/vespa/config/Connection.java +++ b/config/src/main/java/com/yahoo/vespa/config/Connection.java @@ -11,6 +11,8 @@ public interface Connection { void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter); + void invokeSync(Request request, double jrtTimeout); + void setError(int errorCode); void setSuccess(); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java index 8ea2800e65e..5a6f8a8848b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; +import com.yahoo.jrt.Supervisor; + /** * @author hmusum */ @@ -15,4 +17,6 @@ public interface ConnectionPool { Connection setNewCurrentConnection(); int getSize(); + + Supervisor getSupervisor(); } diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java index 96dd6f62244..01da823b87b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java @@ -13,6 +13,7 @@ import java.util.logging.Logger; * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> */ public class JRTConnection implements Connection { + public final static Logger logger = Logger.getLogger(JRTConnection.class.getPackage().getName()); private final String address; private final Supervisor supervisor; @@ -30,17 +31,20 @@ public class JRTConnection implements Connection { yyyyMMddz.setTimeZone(TimeZone.getTimeZone("GMT")); } + + public JRTConnection(String address, Supervisor supervisor) { + this.address = address; + this.supervisor = supervisor; + } + @Override public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { getTarget().invokeAsync(request, jrtTimeout, requestWaiter); } - public final static Logger logger = Logger.getLogger(JRTConnection.class.getPackage().getName()); - - - public JRTConnection(String address, Supervisor supervisor) { - this.address = address; - this.supervisor = supervisor; + @Override + public void invokeSync(Request request, double jrtTimeout) { + getTarget().invokeSync(request, jrtTimeout); } public String getAddress() { diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index b27f83851b4..efeaacf225b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -21,10 +21,11 @@ import java.util.logging.Logger; * The current connection is available with {@link #getCurrent()}. * When calling {@link #setError(Connection, int)}, {#link #setNewCurrentConnection} will always be called. * - * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author Gunnar Gauslaa Bergem * @author hmusum */ public class JRTConnectionPool implements ConnectionPool { + private static final Logger log = Logger.getLogger(JRTConnectionPool.class.getName()); private final Supervisor supervisor = new Supervisor(new Transport()); @@ -150,4 +151,9 @@ public class JRTConnectionPool implements ConnectionPool { } } + @Override + public Supervisor getSupervisor() { + return supervisor; + } + } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java index 2aa7c66ce87..8acab56d838 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java @@ -130,4 +130,32 @@ public class ConfigSetSubscriptionTest { assertEquals(hS.getConfig().stringVal(), "new StringVal"); } + @Test + public void requireThatWeGetLatestConfigWhenTwoUpdatesBeforeClientChecks() { + ConfigSet myConfigs = new ConfigSet(); + AppConfig.Builder a0builder = new AppConfig.Builder().message("A message, 1"); + myConfigs.addBuilder("app/0", a0builder); + ConfigSubscriber subscriber = new ConfigSubscriber(myConfigs); + ConfigHandle<AppConfig> hA0 = subscriber.subscribe(AppConfig.class, "app/0"); + + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertEquals(hA0.getConfig().message(), "A message, 1"); + + assertFalse(subscriber.nextConfig(10)); + assertFalse(hA0.isChanged()); + assertEquals(hA0.getConfig().message(), "A message, 1"); + + //Reconfigure two times in a row + a0builder.message("A new message, 2"); + subscriber.reload(1); + a0builder.message("An even newer message, 3"); + subscriber.reload(2); + + // Should pick up the last one + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertEquals(hA0.getConfig().message(), "An even newer message, 3"); + } + } diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 51c29823664..f6268c8a84a 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -38,10 +38,8 @@ ConfigSubscriptionSet::acquireSnapshot(uint64_t timeoutInMillis, bool ignoreChan int64_t lastGeneration = _currentGeneration; bool inSync = false; - for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); - it != mt; - it++) { - (*it)->reset(); + for (const auto & subscription : _subscriptionList) { + subscription->reset(); } LOG(debug, "Going into nextConfig loop, time left is %d", timeLeft); @@ -52,10 +50,7 @@ ConfigSubscriptionSet::acquireSnapshot(uint64_t timeoutInMillis, bool ignoreChan int64_t generation = -1; // Run nextUpdate on all subscribers to get them in sync. - for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); - it != mt; - it++) { - ConfigSubscription::SP subscription = *it; + for (const auto & subscription : _subscriptionList) { if (!subscription->nextUpdate(_currentGeneration, timeLeft)) break; @@ -91,10 +86,14 @@ ConfigSubscriptionSet::acquireSnapshot(uint64_t timeoutInMillis, bool ignoreChan LOG(spam, "Config was updated from %ld to %ld", _currentGeneration, lastGeneration); _currentGeneration = lastGeneration; _state = CONFIGURED; - for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); - it != mt; - it++) { - (*it)->flip(); + for (const auto & subscription : _subscriptionList) { + const ConfigKey & key(subscription->getKey()); + LOG(debug, "Updated config id(%s), defname(%s), has changed: %s, lastGenerationChanged: %ld", + key.getConfigId().c_str(), + key.getDefName().c_str(), + (subscription->hasChanged() ? "true" : "false"), + subscription->getLastGenerationChanged()); + subscription->flip(); } } return updated; @@ -104,9 +103,9 @@ void ConfigSubscriptionSet::close() { _state = CLOSED; - for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); it != mt; it++) { - _mgr.unsubscribe(*it); - (*it)->close(); + for (const auto & subscription : _subscriptionList) { + _mgr.unsubscribe(subscription); + subscription->close(); } } diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 9297383c53f..fc9be8ce91b 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -16,6 +16,10 @@ vespa_generate_config(configdefinitions configserver.def) install_config_definition(configserver.def cloud.config.configserver.def) vespa_generate_config(configdefinitions dispatch.def) install_config_definition(dispatch.def vespa.config.search.dispatch.def) +vespa_generate_config(configdefinitions filedistributor.def) +install_config_definition(filedistributor.def cloud.config.filedistribution.filedistributor.def) +vespa_generate_config(configdefinitions filereferences.def) +install_config_definition(filereferences.def cloud.config.filedistribution.filereferences.def) vespa_generate_config(configdefinitions fleetcontroller.def) install_config_definition(fleetcontroller.def vespa.config.content.fleetcontroller.def) vespa_generate_config(configdefinitions ilscripts.def) diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedistributor.def b/configdefinitions/src/vespa/filedistributor.def index 13dd373eac6..13dd373eac6 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/filedistributor.def +++ b/configdefinitions/src/vespa/filedistributor.def diff --git a/filedistribution/src/vespa/filedistribution/model/filereferences.def b/configdefinitions/src/vespa/filereferences.def index 03e071959ce..03e071959ce 100644 --- a/filedistribution/src/vespa/filedistribution/model/filereferences.def +++ b/configdefinitions/src/vespa/filereferences.def diff --git a/configserver/pom.xml b/configserver/pom.xml index 8776fbd5ad1..30d92dc7650 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -126,6 +126,11 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>filedistribution</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>filedistributionmanager</artifactId> <version>${project.version}</version> </dependency> @@ -172,6 +177,11 @@ <artifactId>jersey-proxy-client</artifactId> </dependency> <dependency> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + <scope>compile</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/PathProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/PathProvider.java deleted file mode 100644 index 5910c69048a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/PathProvider.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server; - -import com.google.inject.Inject; -import com.yahoo.path.Path; - -/** - * Temporary provider of root path for components that will soon get them injected from a parent class. - * - * @author lulf - * * @since 5.1.24 - */ -public class PathProvider { - - private final Path root; - // Path for Vespa-related data stored in Zookeeper (subpaths are relative to this path) - // NOTE: This should not be exposed, as this path can be different in testing, depending on how we configure it. - private static final String APPS_ZK_NODE = "sessions"; - private static final String VESPA_ZK_PATH = "/vespa/config"; - - @Inject - public PathProvider() { - root = Path.fromString(VESPA_ZK_PATH); - } - - public PathProvider(Path root) { - this.root = root; - } - - public Path getRoot() { - return root; - } - - public Path getSessionDirs() { - return root.append(APPS_ZK_NODE); - } - - public Path getSessionDir(long sessionId) { - return getSessionDirs().append(String.valueOf(sessionId)); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index 648e6bb7180..c3d13b86591 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -18,11 +18,15 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** @@ -36,10 +40,13 @@ import java.util.logging.Logger; public class ZKTenantApplications implements TenantApplications, PathChildrenCacheListener { private static final Logger log = Logger.getLogger(ZKTenantApplications.class.getName()); + private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); + private final Curator curator; private final Path applicationsPath; private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKTenantApplications.class.getName())); + private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final TenantName tenant; @@ -47,16 +54,21 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac private ZKTenantApplications(Curator curator, Path applicationsPath, ReloadHandler reloadHandler, TenantName tenant) { this.curator = curator; this.applicationsPath = applicationsPath; + curator.create(applicationsPath); this.reloadHandler = reloadHandler; this.tenant = tenant; this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); this.directoryCache.start(); this.directoryCache.addListener(this); + checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeApplications, + checkForRemovedApplicationsInterval.getSeconds(), + checkForRemovedApplicationsInterval.getSeconds(), + TimeUnit.SECONDS); } - public static TenantApplications create(Curator curator, Path applicationsPath, ReloadHandler reloadHandler, TenantName tenant) { + public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { try { - return new ZKTenantApplications(curator, applicationsPath, reloadHandler, tenant); + return new ZKTenantApplications(curator, Tenants.getApplicationsPath(tenant), reloadHandler, tenant); } catch (Exception e) { throw new RuntimeException(Tenants.logPre(tenant) + "Error creating application repo", e); } @@ -118,6 +130,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac public void close() { directoryCache.close(); pathChildrenExecutor.shutdown(); + checkForRemovedApplicationsService.shutdown(); } @Override @@ -138,22 +151,22 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac } // We might have lost events and might need to remove applications (new applications are // not added by listening for events here, they are added when session is added, see RemoteSessionRepo) - removeApplications(event.getType()); + removeApplications(); } private void applicationRemoved(ApplicationId applicationId) { reloadHandler.removeApplication(applicationId); - log.log(LogLevel.DEBUG, Tenants.logPre(applicationId) + "Application removed: " + applicationId); + log.log(LogLevel.INFO, Tenants.logPre(applicationId) + "Application removed: " + applicationId); } private void applicationAdded(ApplicationId applicationId) { log.log(LogLevel.DEBUG, Tenants.logPre(applicationId) + "Application added: " + applicationId); } - private void removeApplications(PathChildrenCacheEvent.Type eventType) { + private void removeApplications() { ImmutableSet<ApplicationId> activeApplications = ImmutableSet.copyOf(listApplications()); - log.log(LogLevel.DEBUG, "Got " + eventType + " event for tenant '" + tenant + - "', removing applications except these active applications: " + activeApplications); + log.log(LogLevel.DEBUG, "Removing stale applications for tenant '" + tenant + + "', not removing these active applications: " + activeApplications); reloadHandler.removeApplicationsExcept(activeApplications); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 70b677b4057..c0c9c309576 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -2,11 +2,16 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.component.Version; -import com.yahoo.config.model.api.*; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; @@ -132,6 +137,7 @@ public class ModelContextImpl implements ModelContext { private final ApplicationId applicationId; private final boolean multitenant; private final List<ConfigServerSpec> configServerSpecs; + private final HostName loadBalancerName; private final boolean hostedVespa; private final Zone zone; private final Set<Rotation> rotations; @@ -139,12 +145,14 @@ public class ModelContextImpl implements ModelContext { public Properties(ApplicationId applicationId, boolean multitenant, List<ConfigServerSpec> configServerSpecs, + HostName loadBalancerName, boolean hostedVespa, Zone zone, Set<Rotation> rotations) { this.applicationId = applicationId; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; + this.loadBalancerName = loadBalancerName; this.hostedVespa = hostedVespa; this.zone = zone; this.rotations = rotations; @@ -166,6 +174,11 @@ public class ModelContextImpl implements ModelContext { } @Override + public HostName loadBalancerName() { + return loadBalancerName; + } + + @Override public boolean hostedVespa() { return hostedVespa; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java new file mode 100644 index 00000000000..61c376a7256 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -0,0 +1,9 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; + +public interface AddFileInterface { + FileReference addFile(String relativePath); + FileReference addFile(String relativePath, FileReference reference); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java new file mode 100644 index 00000000000..0d1aae97690 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -0,0 +1,25 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import java.io.File; + +public class ApplicationFileManager implements AddFileInterface { + private final File applicationDir; + private final FileDirectory master; + + ApplicationFileManager(File applicationDir, FileDirectory master) { + this.applicationDir = applicationDir; + this.master = master; + } + + @Override + public FileReference addFile(String relativePath, FileReference reference) { + return master.addFile(new File(applicationDir, relativePath), reference); + } + + @Override + public FileReference addFile(String relativePath) { + return master.addFile(new File(applicationDir, relativePath)); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java new file mode 100644 index 00000000000..588f2d1d63f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java @@ -0,0 +1,30 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; + +import java.util.Collection; +import java.util.Set; + +public class CombinedLegacyDistribution implements FileDistribution { + private final FileDistribution legacy; + + CombinedLegacyDistribution(FileDBHandler legacy) { + this.legacy = legacy; + } + @Override + public void sendDeployedFiles(String hostName, Set<FileReference> fileReferences) { + legacy.sendDeployedFiles(hostName, fileReferences); + } + + @Override + public void reloadDeployFileDistributor() { + legacy.reloadDeployFileDistributor(); + } + + @Override + public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { + legacy.removeDeploymentsThatHaveDifferentApplicationId(targetHostnames); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java new file mode 100644 index 00000000000..8f2cb194bbd --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; + +import java.util.List; + +public class CombinedLegacyRegistry implements FileRegistry { + private final FileDBRegistry legacy; + private final FileDBRegistry future; + + CombinedLegacyRegistry(FileDBRegistry legacy, FileDBRegistry future) { + this.legacy = legacy; + this.future = future; + } + @Override + public FileReference addFile(String relativePath) { + FileReference reference = legacy.addFile(relativePath); + return future.addFile(relativePath, reference); + } + + @Override + public String fileSourceHost() { + return future.fileSourceHost(); + } + + @Override + public List<Entry> export() { + return future.export(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java index d7dce9d3f3d..f0ce6104496 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java @@ -31,11 +31,6 @@ public class FileDBHandler implements FileDistribution { } @Override - public void limitSendingOfDeployedFilesTo(Collection<String> hostNames) { - manager.limitSendingOfDeployedFilesTo(hostNames); - } - - @Override public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { manager.removeDeploymentsThatHaveDifferentApplicationId(targetHostnames); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index d921d8d4f8d..1a76454fbed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -4,29 +4,41 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; -import com.yahoo.vespa.filedistribution.FileDistributionManager; -import com.yahoo.config.model.application.provider.FileReferenceCreator; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * @author tonytv */ public class FileDBRegistry implements FileRegistry { - private final FileDistributionManager manager; + private final AddFileInterface manager; private List<Entry> entries = new ArrayList<>(); private final Map<String, FileReference> fileReferenceCache = new HashMap<>(); - public FileDBRegistry(FileDistributionManager manager) { + public FileDBRegistry(AddFileInterface manager) { this.manager = manager; } + public synchronized FileReference addFile(String relativePath, FileReference reference) { + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); + return cachedReference.orElseGet(() -> { + FileReference newRef = manager.addFile(relativePath, reference); + entries.add(new Entry(relativePath, newRef)); + fileReferenceCache.put(relativePath, newRef); + return newRef; + }); + } + @Override public synchronized FileReference addFile(String relativePath) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); return cachedReference.orElseGet(() -> { - FileReference newRef = FileReferenceCreator.create(manager.addFile(relativePath)); + FileReference newRef = manager.addFile(relativePath); entries.add(new Entry(relativePath, newRef)); fileReferenceCache.put(relativePath, newRef); return newRef; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java new file mode 100644 index 00000000000..7d0ba6cd9bd --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -0,0 +1,120 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.io.IOUtils; +import com.yahoo.text.Utf8; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.logging.Logger; + +public class FileDirectory { + private static final Logger log = Logger.getLogger(FileDirectory.class.getName()); + private final File root; + + public FileDirectory() { + this(FileDistribution.getDefaultFileDBPath()); + } + + public FileDirectory(File rootDir) { + root = rootDir; + try { + ensureRootExist(); + } catch (IllegalArgumentException e) { + log.warning("Failed creating directory in constructor, will retry on demand : " + e.toString()); + } + } + + private void ensureRootExist() { + if (! root.exists()) { + if ( ! root.mkdir()) { + throw new IllegalArgumentException("Failed creating root dir '" + root.getAbsolutePath() + "'."); + } + } else if (!root.isDirectory()) { + throw new IllegalArgumentException("'" + root.getAbsolutePath() + "' is not a directory"); + } + } + + static private class Filter implements FilenameFilter { + @Override + public boolean accept(File dir, String name) { + return !".".equals(name) && !"..".equals(name) ; + } + } + + String getPath(FileReference ref) { + return root.getAbsolutePath() + "/" + ref.value(); + } + + File getFile(FileReference reference) { + ensureRootExist(); + File dir = new File(getPath(reference)); + if (!dir.exists()) { + throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + "' does not exist."); + } + if (!dir.isDirectory()) { + throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + "' is not a directory."); + } + File [] files = dir.listFiles(new Filter()); + if (files.length != 1) { + StringBuilder msg = new StringBuilder(); + for (File f: files) { + msg.append(f.getName()).append("\n"); + } + throw new IllegalArgumentException("File reference '" + reference.toString() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain exactly one file, but [" + msg.toString() + "]"); + } + return files[0]; + } + + private Long computeReference(File file) throws IOException { + byte [] wholeFile = IOUtils.readFileBytes(file); + XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0)); + } + + public FileReference addFile(File source) { + try { + Long hash = computeReference(source); + FileReference reference = new FileReference(Long.toHexString(hash)); + return addFile(source, reference); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + public FileReference addFile(File source, FileReference reference) { + ensureRootExist(); + try { + File destinationDir = new File(root, reference.value()); + if (!destinationDir.exists()) { + destinationDir.mkdir(); + Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing"); + File destination = new File(tempDestinationDir.toFile(), source.getName()); + if (source.isDirectory()) + IOUtils.copyDirectory(source, destination); + else + IOUtils.copy(source, destination); + if (!destinationDir.exists()) { + if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { + log.warning("Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); + } + } else { + IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, 1); + } + IOUtils.recursiveDeleteDir(tempDestinationDir.toFile()); + } + return reference; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java index 3693bfb361c..59c3a54897d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.vespa.filedistribution.FileDistributionManager; @@ -19,16 +20,32 @@ public class FileDistributionProvider { private final FileRegistry fileRegistry; private final FileDistribution fileDistribution; - public FileDistributionProvider(File applicationDir, String zooKeepersSpec, String applicationId, Lock fileDistributionLock) { + static private class ManagerWrapper implements AddFileInterface { + private final FileDistributionManager manager; + ManagerWrapper(FileDistributionManager manager) { + this.manager = manager; + } + @Override + public FileReference addFile(String relativePath) { + return new FileReference(manager.addFile(relativePath)); + } + + @Override + public FileReference addFile(String relativePath, FileReference reference) { + throw new IllegalStateException("addFile with external reference is not possible with legacy filedistribution."); + } + } + + public FileDistributionProvider(File applicationDir, String zooKeepersSpec, + String applicationId, Lock fileDistributionLock) + { ensureDirExists(FileDistribution.getDefaultFileDBPath()); final FileDistributionManager manager = new FileDistributionManager( - FileDistribution.getDefaultFileDBPath(), - applicationDir, - zooKeepersSpec, - applicationId, - fileDistributionLock); - this.fileDistribution = new FileDBHandler(manager); - this.fileRegistry = new FileDBRegistry(manager); + FileDistribution.getDefaultFileDBPath(), applicationDir, + zooKeepersSpec, applicationId, fileDistributionLock); + this.fileDistribution = new CombinedLegacyDistribution(new FileDBHandler(manager)); + this.fileRegistry = new CombinedLegacyRegistry(new FileDBRegistry(new ManagerWrapper(manager)), + new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory()))); } public FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java new file mode 100644 index 00000000000..9316a9a5c8e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -0,0 +1,145 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.net.HostName; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.filedistribution.FileDownloader; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +public class FileServer { + private static final Logger log = Logger.getLogger(FileServer.class.getName()); + private final FileDirectory root; + private final ExecutorService executor; + private final FileDownloader downloader; + + public static class ReplayStatus { + private final int code; + private final String description; + public ReplayStatus(int code, String description) { + this.code = code; + this.description = description; + } + public boolean ok() { return code == 0; } + public int getCode() { return code; } + public String getDescription() { return description; } + } + + public interface Receiver { + void receive(FileReference reference, String filename, byte [] content, ReplayStatus status); + } + + @Inject + public FileServer(ConfigserverConfig configserverConfig) { + this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath()); + } + + // For testing only + public FileServer(File rootDir) { + this(new EmptyConnectionPool(), rootDir); + } + + private FileServer(ConnectionPool connectionPool, File rootDir) { + this.downloader = new FileDownloader(connectionPool); + this.root = new FileDirectory(rootDir); + this.executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + } + + public boolean hasFile(String fileName) { + return hasFile(new FileReference(fileName)); + } + public boolean hasFile(FileReference reference) { + try { + return root.getFile(reference).exists(); + } catch (IllegalArgumentException e) { + log.warning("Failed locating file reference '" + reference + "' with error " + e.toString()); + } + return false; + } + public boolean startFileServing(String fileName, Receiver target) { + FileReference reference = new FileReference(fileName); + File file = root.getFile(reference); + + if (file.exists()) { + executor.execute(() -> serveFile(reference, target)); + } + return false; + } + + private void serveFile(FileReference reference, Receiver target) { + File file = root.getFile(reference); + // TODO remove once verified in system tests. + log.info("Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + byte [] blob = new byte [0]; + boolean success = false; + String errorDescription = "OK"; + try { + blob = IOUtils.readFileBytes(file); + success = true; + } catch (IOException e) { + errorDescription = "For file reference '" + reference.value() + "' I failed reading file '" + file.getAbsolutePath() + "'"; + log.warning(errorDescription + "for sending to '" + target.toString() + "'. " + e.toString()); + } + target.receive(reference, file.getName(), blob, + new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); + // TODO remove once verified in system tests. + log.info("Done serving reference '" + reference.toString() + "' with file '" + file.getAbsolutePath() + "'"); + } + + public void download(FileReference fileReference) { + downloader.getFile(fileReference); + } + + public FileDownloader downloader() { + return downloader; + } + + // Connection pool with all config servers except this one (might be an empty pool if there is only one config server) + private static ConnectionPool createConnectionPool(ConfigserverConfig configserverConfig) { + List<String> configServers = ConfigServerSpec.fromConfig(configserverConfig) + .stream() + .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost())) + .map(spec -> "tcp/" + spec.getHostName() + ":" + spec.getConfigServerPort()) + .collect(Collectors.toList()); + + return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : new EmptyConnectionPool(); + } + + private static class EmptyConnectionPool implements ConnectionPool { + + @Override + public void close() {} + + @Override + public void setError(Connection connection, int i) {} + + @Override + public Connection getCurrent() { return null; } + + @Override + public Connection setNewCurrentConnection() { return null; } + + @Override + public int getSize() { return 0; } + + @Override + public Supervisor getSupervisor() { return new Supervisor(new Transport()); } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java index 81c777df393..eb23d38e23e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java @@ -14,7 +14,6 @@ import java.util.Set; public class MockFileDBHandler implements FileDistribution { public int sendDeployedFilesCalled = 0; public int reloadDeployFileDistributorCalled = 0; - public int limitSendingOfDeployedFilesToCalled = 0; public int removeDeploymentsThatHaveDifferentApplicationIdCalled = 0; @Override @@ -28,11 +27,6 @@ public class MockFileDBHandler implements FileDistribution { } @Override - public void limitSendingOfDeployedFilesTo(Collection<String> hostNames) { - limitSendingOfDeployedFilesToCalled++; - } - - @Override public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { removeDeploymentsThatHaveDifferentApplicationIdCalled++; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java index 873a24b5f05..e5bf8e22020 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java @@ -31,9 +31,10 @@ public class Utils { com.yahoo.jdisc.http.HttpRequest jDiscRequest = req.getJDiscRequest(); BindingMatch<?> bm = jDiscRequest.getBindingMatch(); if (bm == null) { + UriPattern pattern = new UriPattern(uriPattern); bm = new BindingMatch<>( - new UriPattern(uriPattern).match(URI.create(jDiscRequest.getUri().toString())), - new Object()); + pattern.match(URI.create(jDiscRequest.getUri().toString())), + new Object(), pattern); } return bm; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java index db92f53aacd..59270afd397 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java @@ -39,7 +39,7 @@ public class HttpConfigRequests { UriPattern fullAppIdPattern = new UriPattern(pattern); URI uri = req.getUri(); Match match = fullAppIdPattern.match(uri); - if (match!=null) return new BindingMatch<>(match, new Object()); + if (match!=null) return new BindingMatch<>(match, new Object(), fullAppIdPattern); } throw new IllegalArgumentException("Illegal url for config request: " + req.getUri()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 1806414f510..85249d4e87d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -75,7 +75,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { serviceInfo.getServiceType().equals("qrserver")). findAny(); if (container.isPresent()) { - activeRotation |= Boolean.valueOf(container.get().getProperty("activeRotation").get()); + activeRotation |= Boolean.valueOf(container.get().getProperty("activeRotation").orElse("false")); } } return activeRotation; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index af4d998c347..48732814919 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -6,16 +6,19 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.OutOfCapacityException; -import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; +import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; import com.yahoo.vespa.config.server.provision.StaticProvisioner; @@ -24,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -96,7 +100,12 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { catch (RuntimeException e) { boolean isOldestMajor = i == majorVersions.size() - 1; if (isOldestMajor) { - throw new IllegalArgumentException(applicationId + ": Error loading model", e); + if (e instanceof NullPointerException || e instanceof NoSuchElementException) { + log.log(LogLevel.WARNING, "Unexpected error when building model ", e); + throw new InternalServerException(applicationId + ": Error loading model", e); + } else { + throw new IllegalArgumentException(applicationId + ": Error loading model", e); + } } else { log.log(Level.INFO, applicationId + ": Skipping major version " + majorVersions.get(i), e); } @@ -170,9 +179,10 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { ConfigserverConfig configserverConfig, Zone zone, Set<Rotation> rotations) { - return new ModelContextImpl.Properties(applicationId, + return new ModelContextImpl.Properties(applicationId, configserverConfig.multitenant(), ConfigServerSpec.fromConfig(configserverConfig), + HostName.from(configserverConfig.loadBalancerAddress()), configserverConfig.hostedVespa(), zone, rotations); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index ffd86ac121e..8aaf053247e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -76,7 +76,8 @@ public class DelayedConfigResponses { } public synchronized void run() { - remove(); + removeFromQueue(); + removeWatcher(); rpcServer.addToRequestQueue(request, true, null); if (log.isLoggable(LogLevel.DEBUG)) { log.log(LogLevel.DEBUG, logPre()+"DelayedConfigResponse. putting on queue: " + request.getShortDescription()); @@ -86,9 +87,8 @@ public class DelayedConfigResponses { /** * Remove delayed response from its queue */ - private void remove() { + private void removeFromQueue() { delayedResponsesQueue.remove(this); - removeWatcher(); } public JRTServerConfigRequest getRequest() { @@ -107,8 +107,13 @@ public class DelayedConfigResponses { return Tenants.logPre(app); } - public synchronized boolean cancel() { - remove(); + synchronized void cancelAndRemove() { + removeFromQueue(); + cancel(); + } + + synchronized boolean cancel() { + removeWatcher(); if (future == null) { throw new IllegalStateException("Cannot cancel a task that has not been scheduled"); } @@ -129,7 +134,7 @@ public class DelayedConfigResponses { */ @Override public void notifyTargetInvalid(Target target) { - cancel(); + cancelAndRemove(); } private void addWatcher() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 3c9917bf17e..d17cdf722ea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -4,18 +4,22 @@ package com.yahoo.vespa.config.server.rpc; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int64Value; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.Method; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ErrorCode; @@ -27,6 +31,7 @@ import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.SuperModelRequestHandler; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.GetConfigContext; +import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.ReloadListener; @@ -36,7 +41,10 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory; import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; import com.yahoo.vespa.config.server.tenant.TenantListener; import com.yahoo.vespa.config.server.tenant.Tenants; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -68,6 +76,18 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { static final int TRACELEVEL_DEBUG = 9; private static final String THREADPOOL_NAME = "rpcserver worker pool"; private static final long SHUTDOWN_TIMEOUT = 60; + private enum FileApiErrorCodes { + OK(0, "OK"), + NOT_FOUND(1, "Filereference not found"); + private final int code; + private final String description; + FileApiErrorCodes(int code, String description) { + this.code = code; + this.description = description; + } + int getCode() { return code; } + String getDescription() { return description; } + } private final Supervisor supervisor = new Supervisor(new Transport()); private Spec spec = null; private final boolean useRequestVersion; @@ -83,6 +103,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private final MetricUpdater metrics; private final MetricUpdaterFactory metricUpdaterFactory; private final HostLivenessTracker hostLivenessTracker; + private final FileServer fileServer; private final ThreadPoolExecutor executorService; private volatile boolean allTenantsLoaded = false; @@ -93,20 +114,23 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { * @param config The config to use for setting up this server */ @Inject - public RpcServer(ConfigserverConfig config, SuperModelRequestHandler superModelRequestHandler, MetricUpdaterFactory metrics, - HostRegistries hostRegistries, HostLivenessTracker hostLivenessTracker) { + public RpcServer(ConfigserverConfig config, SuperModelRequestHandler superModelRequestHandler, + MetricUpdaterFactory metrics, HostRegistries hostRegistries, + HostLivenessTracker hostLivenessTracker, FileServer fileServer) { this.superModelRequestHandler = superModelRequestHandler; - this.metricUpdaterFactory = metrics; - this.supervisor.setMaxOutputBufferSize(config.maxoutputbuffersize()); + metricUpdaterFactory = metrics; + supervisor.setMaxOutputBufferSize(config.maxoutputbuffersize()); this.metrics = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); this.hostLivenessTracker = hostLivenessTracker; BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); - executorService = new ThreadPoolExecutor(config.numthreads(), config.numthreads(), 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); + executorService = new ThreadPoolExecutor(config.numthreads(), config.numthreads(), + 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); spec = new Spec(null, config.rpcport()); hostRegistry = hostRegistries.getTenantHostRegistry(); this.useRequestVersion = config.useVespaVersionInRequest(); this.hostedVespa = config.hostedVespa(); + this.fileServer = fileServer; setUpHandlers(); } @@ -180,6 +204,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { getSupervisor().addMethod(new Method("printStatistics", "", "s", this, "printStatistics") .methodDesc("printStatistics") .returnDesc(0, "statistics", "Statistics for server")); + getSupervisor().addMethod(new Method("filedistribution.serveFile", "s", "is", this, "serveFile")); } /** @@ -223,7 +248,6 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } for (int i = 0; i < responsesSent; i++) { - try { completionService.take(); } catch (InterruptedException e) { @@ -402,4 +426,56 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { return useRequestVersion; } + class FileReceiver implements FileServer.Receiver { + Target target; + FileReceiver(Target target) { + this.target = target; + } + + @Override + public String toString() { + return target.toString(); + } + + @Override + public void receive(FileReference reference, String filename, byte [] content, FileServer.ReplayStatus status) { + XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + Request fileBlob = new Request("filedistribution.receiveFile"); + fileBlob.parameters().add(new StringValue(reference.value())); + fileBlob.parameters().add(new StringValue(filename)); + fileBlob.parameters().add(new DataValue(content)); + fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); + fileBlob.parameters().add(new Int32Value(status.getCode())); + fileBlob.parameters().add(new StringValue(status.getDescription())); + target.invokeSync(fileBlob, 600); + if (fileBlob.isError()) { + log.warning("Failed delivering reference '" + reference.value() + "' with file '" + filename + "' to " + + target.toString() + " with error : '" + fileBlob.errorMessage() + "'."); + } + } + } + + @SuppressWarnings("UnusedDeclaration") + public final void serveFile(Request request) { + String fileReference = request.parameters().get(0).asString(); + FileApiErrorCodes result; + try { + // TODO remove once verified in system tests. + log.info("Received request for reference '" + fileReference + "'"); + result = fileServer.hasFile(fileReference) + ? FileApiErrorCodes.OK + : FileApiErrorCodes.NOT_FOUND; + if (result == FileApiErrorCodes.OK) { + fileServer.startFileServing(fileReference, new FileReceiver(request.target())); + } else { + fileServer.download(new FileReference(fileReference)); + } + } catch (IllegalArgumentException e) { + result = FileApiErrorCodes.NOT_FOUND; + log.warning("Failed serving file reference '" + fileReference + "' with error " + e.toString()); + } + request.returnValues() + .add(new Int32Value(result.getCode())) + .add(new StringValue(result.getDescription())); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java index 298acaca901..e96ddb4b094 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java @@ -6,6 +6,7 @@ import com.yahoo.path.Path; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; @@ -19,20 +20,19 @@ public class RemoteSessionFactory { private final GlobalComponentRegistry componentRegistry; private final Curator curator; private final ConfigCurator configCurator; - private final Path sessionDirPath; + private final Path sessionsPath; private final ConfigDefinitionRepo defRepo; private final TenantName tenant; private final ConfigserverConfig configserverConfig; private final Clock clock; public RemoteSessionFactory(GlobalComponentRegistry componentRegistry, - Path sessionsPath, TenantName tenant, Clock clock) { this.componentRegistry = componentRegistry; this.curator = componentRegistry.getCurator(); this.configCurator = componentRegistry.getConfigCurator(); - this.sessionDirPath = sessionsPath; + this.sessionsPath = Tenants.getSessionsPath(tenant); this.tenant = tenant; this.defRepo = componentRegistry.getConfigDefinitionRepo(); this.configserverConfig = componentRegistry.getConfigserverConfig(); @@ -40,7 +40,7 @@ public class RemoteSessionFactory { } public RemoteSession createSession(long sessionId) { - Path sessionPath = sessionDirPath.append(String.valueOf(sessionId)); + Path sessionPath = this.sessionsPath.append(String.valueOf(sessionId)); SessionZooKeeperClient sessionZKClient = new SessionZooKeeperClient(curator, configCurator, sessionPath, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 659a44bb339..2269a7ed997 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -8,10 +8,12 @@ import java.util.logging.Logger; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.Curator; import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.server.ReloadHandler; @@ -49,19 +51,19 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod * @param curator a {@link Curator} instance. * @param remoteSessionFactory a {@link com.yahoo.vespa.config.server.session.RemoteSessionFactory} * @param reloadHandler a {@link com.yahoo.vespa.config.server.ReloadHandler} - * @param sessionsPath a {@link com.yahoo.path.Path} to the sessions dir. - * @param applicationRepo an {@link TenantApplications} object. + * @param tenant a {@link TenantName} instance. + * @param applicationRepo a {@link TenantApplications} instance. * @param executorService an {@link ExecutorService} to run callbacks from ZooKeeper. */ public RemoteSessionRepo(Curator curator, - RemoteSessionFactory remoteSessionFactory, - ReloadHandler reloadHandler, - Path sessionsPath, - TenantApplications applicationRepo, - MetricUpdater metricUpdater, - ExecutorService executorService) { + RemoteSessionFactory remoteSessionFactory, + ReloadHandler reloadHandler, + TenantName tenant, + TenantApplications applicationRepo, + MetricUpdater metricUpdater, + ExecutorService executorService) { this.curator = curator; - this.sessionsPath = sessionsPath; + this.sessionsPath = Tenants.getSessionsPath(tenant); this.applicationRepo = applicationRepo; this.remoteSessionFactory = remoteSessionFactory; this.reloadHandler = reloadHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 1d5025f2e61..fdc681b5fb6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.host.HostValidator; +import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -57,7 +58,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { public SessionFactoryImpl(GlobalComponentRegistry globalComponentRegistry, SessionCounter sessionCounter, - Path sessionsPath, TenantApplications applicationRepo, TenantFileSystemDirs tenantFileSystemDirs, HostValidator<ApplicationId> hostRegistry, @@ -68,7 +68,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { this.curator = globalComponentRegistry.getCurator(); this.configCurator = globalComponentRegistry.getConfigCurator(); this.sessionCounter = sessionCounter; - this.sessionsPath = sessionsPath; + this.sessionsPath = Tenants.getSessionsPath(tenant); this.applicationRepo = applicationRepo; this.tenantFileSystemDirs = tenantFileSystemDirs; this.superModelGenerationCounter = globalComponentRegistry.getSuperModelGenerationCounter(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 31be18d9b22..531085883c4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -9,25 +9,32 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.http.InvalidApplicationException; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; - +import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.curator.Curator; import org.xml.sax.SAXException; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; import java.io.IOException; import java.time.Instant; import java.util.List; @@ -37,9 +44,6 @@ import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; - /** * A SessionPreparer is responsible for preparing a session given an application package. * @@ -148,6 +152,7 @@ public class SessionPreparer { this.properties = new ModelContextImpl.Properties(params.getApplicationId(), configserverConfig.multitenant(), ConfigServerSpec.fromConfig(configserverConfig), + HostName.from(configserverConfig.loadBalancerAddress()), configserverConfig.hostedVespa(), zone, rotationsSet); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 084d35a42d4..61145c2a138 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -34,7 +34,6 @@ public class TenantBuilder { private final Path tenantPath; private final GlobalComponentRegistry componentRegistry; private final TenantName tenant; - private final Path sessionsPath; private RemoteSessionRepo remoteSessionRepo; private LocalSessionRepo localSessionRepo; private SessionFactory sessionFactory; @@ -47,15 +46,14 @@ public class TenantBuilder { private TenantFileSystemDirs tenantFileSystemDirs; private HostValidator<ApplicationId> hostValidator; - private TenantBuilder(GlobalComponentRegistry componentRegistry, TenantName tenant, Path zkPath) { + private TenantBuilder(GlobalComponentRegistry componentRegistry, TenantName tenant) { this.componentRegistry = componentRegistry; - this.tenantPath = zkPath; + this.tenantPath = Tenants.getTenantPath(tenant); this.tenant = tenant; - this.sessionsPath = tenantPath.append(Tenant.SESSIONS); } - public static TenantBuilder create(GlobalComponentRegistry componentRegistry, TenantName tenant, Path zkPath) { - return new TenantBuilder(componentRegistry, tenant, zkPath); + public static TenantBuilder create(GlobalComponentRegistry componentRegistry, TenantName tenant) { + return new TenantBuilder(componentRegistry, tenant); } public TenantBuilder withSessionFactory(SessionFactory sessionFactory) { @@ -123,7 +121,7 @@ public class TenantBuilder { private void createSessionFactory() { if (sessionFactory == null || localSessionLoader == null) { - SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, sessionCounter, sessionsPath, + SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, sessionCounter, applicationRepo, tenantFileSystemDirs, hostValidator, tenant); if (sessionFactory == null) { sessionFactory = impl; @@ -136,13 +134,13 @@ public class TenantBuilder { private void createApplicationRepo() { if (applicationRepo == null) { - applicationRepo = ZKTenantApplications.create(componentRegistry.getCurator(), tenantPath.append(Tenant.APPLICATIONS), reloadHandler, tenant); + applicationRepo = ZKTenantApplications.create(componentRegistry.getCurator(), reloadHandler, tenant); } } private void createSessionCounter() { if (sessionCounter == null) { - sessionCounter = new SessionCounter(componentRegistry.getCurator(), tenantPath, sessionsPath); + sessionCounter = new SessionCounter(componentRegistry.getCurator(), tenant); } } @@ -167,7 +165,7 @@ public class TenantBuilder { private void createRemoteSessionFactory(Clock clock) { if (remoteSessionFactory == null) { - remoteSessionFactory = new RemoteSessionFactory(componentRegistry, sessionsPath, tenant, clock); + remoteSessionFactory = new RemoteSessionFactory(componentRegistry, tenant, clock); } } @@ -176,7 +174,7 @@ public class TenantBuilder { remoteSessionRepo = new RemoteSessionRepo(componentRegistry.getCurator(), remoteSessionFactory, reloadHandler, - sessionsPath, + tenant, applicationRepo, componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant)), createSingleThreadedExecutorService(RemoteSessionRepo.class.getName())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 60200e34cdf..d2cf17a38d4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -11,7 +11,6 @@ import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -78,10 +77,10 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen * @throws Exception is creating the Tenants instance fails */ @Inject - public Tenants(GlobalComponentRegistry globalComponentRegistry, Metrics metrics) throws Exception { + public Tenants(GlobalComponentRegistry globalComponentRegistry) throws Exception { this.globalComponentRegistry = globalComponentRegistry; this.curator = globalComponentRegistry.getCurator(); - metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); + metricUpdater = globalComponentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(globalComponentRegistry.getTenantListener()); curator.framework().getConnectionStateListenable().addListener(this); @@ -99,15 +98,16 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen /** * New instance containing the given tenants. This will not create Zookeeper watches. For testing only * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} instance - * @param metrics a {@link com.yahoo.vespa.config.server.monitoring.Metrics} instance * @param tenants a collection of {@link Tenant}s */ - public Tenants(GlobalComponentRegistry globalComponentRegistry, Metrics metrics, Collection<Tenant> tenants) { + public Tenants(GlobalComponentRegistry globalComponentRegistry, Collection<Tenant> tenants) { this.globalComponentRegistry = globalComponentRegistry; this.curator = globalComponentRegistry.getCurator(); - metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); + metricUpdater = globalComponentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(globalComponentRegistry.getTenantListener()); curator.create(tenantsPath); + createSystemTenants(globalComponentRegistry.getConfigserverConfig()); + createTenants(); this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); this.tenants.putAll(addTenants(tenants)); } @@ -147,7 +147,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen return tenants; } - synchronized void createTenants() throws Exception { + synchronized void createTenants() { Set<TenantName> allTenants = readTenantsFromZooKeeper(); log.log(LogLevel.DEBUG, "Create tenants, tenants found in zookeeper: " + allTenants); checkForRemovedTenants(allTenants); @@ -159,7 +159,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen Map<TenantName, Tenant> current = new LinkedHashMap<>(tenants); for (Map.Entry<TenantName, Tenant> entry : current.entrySet()) { TenantName tenant = entry.getKey(); - if (!newTenants.contains(tenant)) { + if (!newTenants.contains(tenant) && !DEFAULT_TENANT.equals(tenant)) { notifyRemovedTenant(tenant); entry.getValue().close(); tenants.remove(tenant); @@ -167,7 +167,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen } } - private void checkForAddedTenants(Set<TenantName> newTenants) throws Exception { + private void checkForAddedTenants(Set<TenantName> newTenants) { ExecutorService executor = Executors.newFixedThreadPool(globalComponentRegistry.getConfigserverConfig().numParallelTenantLoaders()); for (TenantName tenantName : newTenants) { // Note: the http handler will check if the tenant exists, and throw accordingly @@ -178,12 +178,18 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen } } executor.shutdown(); - executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen + try { + executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen + } catch (InterruptedException e) { + throw new RuntimeException("Executor for creating tenants did not terminate within timeout"); + } } private void createTenant(TenantName tenantName) { + if (tenants.containsKey(tenantName)) return; + try { - Tenant tenant = TenantBuilder.create(globalComponentRegistry, tenantName, getTenantPath(tenantName)).build(); + Tenant tenant = TenantBuilder.create(globalComponentRegistry, tenantName).build(); notifyNewTenant(tenant); tenants.put(tenantName, tenant); } catch (Exception e) { @@ -251,6 +257,8 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen * @return this Tenants instance */ public synchronized Tenants deleteTenant(TenantName name) { + if (name.equals(DEFAULT_TENANT)) + throw new IllegalArgumentException("Deleting 'default' tenant is not allowed"); Tenant tenant = tenants.get(name); tenant.delete(); return this; @@ -267,7 +275,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen * @return the log string */ public static String logPre(ApplicationId app) { - if (TenantName.defaultName().equals(app.tenant())) return ""; + if (DEFAULT_TENANT.equals(app.tenant())) return ""; StringBuilder ret = new StringBuilder() .append(logPre(app.tenant())) .append("app:"+app.application().value()) @@ -343,6 +351,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen /** * Gets zookeeper path for tenant data + * * @param tenantName tenant name * @return a {@link com.yahoo.path.Path} to the zookeeper data for a tenant */ @@ -350,4 +359,24 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen return tenantsPath.append(tenantName.value()); } + /** + * Gets zookeeper path for session data for a tenant + * + * @param tenantName tenant name + * @return a {@link com.yahoo.path.Path} to the zookeeper sessions data for a tenant + */ + public static Path getSessionsPath(TenantName tenantName) { + return getTenantPath(tenantName).append(Tenant.SESSIONS); + } + + /** + * Gets zookeeper path for application data for a tenant + * + * @param tenantName tenant name + * @return a {@link com.yahoo.path.Path} to the zookeeper application data for a tenant + */ + public static Path getApplicationsPath(TenantName tenantName) { + return getTenantPath(tenantName).append(Tenant.APPLICATIONS); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java index 2d95a013da9..4df292dd204 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java @@ -1,7 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import com.yahoo.path.Path; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.Curator; /** @@ -12,8 +13,10 @@ import com.yahoo.vespa.curator.Curator; */ public class SessionCounter extends InitializedCounter { - public SessionCounter(Curator curator, Path rootPath, Path sessionsDir) { - super(curator, rootPath.append("sessionCounter").getAbsolute(), sessionsDir.getAbsolute()); + public SessionCounter(Curator curator, TenantName tenantName) { + super(curator, + Tenants.getTenantPath(tenantName).append("sessionCounter").getAbsolute(), + Tenants.getSessionsPath(tenantName).getAbsolute()); } /** diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 7ad1b3bbbfd..fbab854ae9e 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -1,13 +1,13 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0" xmlns:preprocess="properties"> - <preprocess:include file='controller/admin.xml' required='false' /> <jdisc id="configserver" jetty="true" version="1.0"> <config name="container.handler.threadpool"> <maxthreads>100</maxthreads> <!-- Reduced thread count to minimize memory consumption --> </config> <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <preprocess:include file='access-logging.xml' required='false' /> <component id="com.yahoo.vespa.config.server.ConfigServerBootstrap" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.monitoring.Metrics" bundle="configserver" /> <component id="com.yahoo.vespa.zookeeper.ZooKeeperServer" bundle="zkfacade" /> @@ -34,6 +34,7 @@ <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> <component id="com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.filedistribution.FileServer" bundle="configserver" /> <component id="com.yahoo.vespa.serviceview.ConfigServerLocation" bundle="configserver" /> @@ -44,6 +45,7 @@ <preprocess:include file='config-models.xml' required='false' /> <preprocess:include file='node-repository.xml' required='false' /> <preprocess:include file='hosted-vespa/routing-status.xml' required='false' /> + <preprocess:include file='hosted-vespa/scoreboard.xml' required='false' /> <preprocess:include file='controller/container.xml' required='false' /> <component id="com.yahoo.vespa.service.monitor.internal.SlobrokMonitorManagerImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.monitor.internal.ServiceMonitorImpl" bundle="service-monitor" /> diff --git a/configserver/src/main/sh/vespa-configserver-remove-state b/configserver/src/main/sh/vespa-configserver-remove-state index 404f0f89a53..c781fcb0c0d 100755 --- a/configserver/src/main/sh/vespa-configserver-remove-state +++ b/configserver/src/main/sh/vespa-configserver-remove-state @@ -75,14 +75,12 @@ usage() { sudo="sudo" ask=true remove_zookeeper_dir=true -remove_applications_dir=true remove_tenants_dir=true confirmed=true zookeeper_dir=var/zookeeper -applications_dir=var/db/vespa/config_server/serverdb/applications tenants_dir=var/db/vespa/config_server/serverdb/tenants -if [ -w $applications_dir ] && [ -w $zookeeper_dir ]; then +if [ -w $zookeeper_dir ]; then sudo="" fi @@ -123,9 +121,8 @@ confirm() { } garbage_collect_dirs() { - find $zookeeper_dir $applications_dir -type d -depth 2>/dev/null | while read dir; do + find $zookeeper_dir $tenants_dir -type d -depth 2>/dev/null | while read dir; do [ "$dir" = "$zookeeper_dir" ] && continue - [ "$dir" = "$applications_dir" ] && continue $sudo rmdir "$dir" 2>/dev/null done } @@ -148,10 +145,6 @@ if $remove_zookeeper_dir && [ -d $zookeeper_dir ]; then confirm_and_clean_dir $zookeeper_dir fi -if $remove_applications_dir && [ -d $applications_dir ]; then - confirm_and_clean_dir $applications_dir -fi - if $remove_tenants_dir && [ -d $tenants_dir ]; then confirm_and_clean_dir $tenants_dir fi diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index 3efad7ac133..dec9dd991de 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -5,8 +5,10 @@ import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -61,7 +63,7 @@ public class InjectedGlobalComponentRegistryTest { serverDB = new ConfigServerDB(configserverConfig); sessionPreparer = new SessionTest.MockSessionPreparer(); rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker()); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); generationCounter = new SuperModelGenerationCounter(curator); defRepo = new StaticConfigDefinitionRepo(); permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index b53f82b25f3..aed0a6a9750 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -45,6 +45,7 @@ public class ModelContextImplTest { ApplicationId.defaultId(), true, Collections.emptyList(), + null, false, Zone.defaultZone(), rotations), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 1a14ac1761c..08cfa74da3b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestWithCurator; +import com.yahoo.vespa.config.server.tenant.Tenants; import org.junit.Test; import java.util.Arrays; @@ -18,60 +18,61 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.*; /** - * @author lulf + * @author Ulf Lilleengen * @since 5.1 */ public class TenantApplicationsTest extends TestWithCurator { + private static final TenantName tenantName = TenantName.from("tenant"); + @Test public void require_that_applications_are_read_from_zookeeper() throws Exception { - curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz", Utf8.toAsciiBytes(3)); - curatorFramework.create().creatingParentsIfNeeded().forPath("/bar:test:bim", Utf8.toAsciiBytes(4)); + writeApplicationData(createApplicationId("foo"), 3L); + writeApplicationData(createApplicationId("bar"), 4L); TenantApplications repo = createZKAppRepo(); List<ApplicationId> applications = repo.listApplications(); assertThat(applications.size(), is(2)); - assertThat(applications.get(0).application().value(), is("dev")); - assertThat(applications.get(1).application().value(), is("test")); + assertThat(applications.get(0).application().value(), is("foo")); + assertThat(applications.get(1).application().value(), is("bar")); assertThat(repo.getSessionIdForApplication(applications.get(0)), is(3L)); assertThat(repo.getSessionIdForApplication(applications.get(1)), is(4L)); } @Test public void require_that_invalid_entries_are_skipped() throws Exception { - curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz"); - curatorFramework.create().creatingParentsIfNeeded().forPath("/invalid"); + writeApplicationData(createApplicationId("foo"), 3L); + writeApplicationData("invalid", 3L); TenantApplications repo = createZKAppRepo(); List<ApplicationId> applications = repo.listApplications(); assertThat(applications.size(), is(1)); - assertThat(applications.get(0).application().value(), is("dev")); + assertThat(applications.get(0).application().value(), is("foo")); } @Test(expected = IllegalArgumentException.class) public void require_that_requesting_session_for_unknown_application_throws_exception() throws Exception { - curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz:bim"); TenantApplications repo = createZKAppRepo(); - repo.getSessionIdForApplication(new ApplicationId.Builder() - .tenant("exist") - .applicationName("tenant").instanceName("here").build()); + repo.getSessionIdForApplication(createApplicationId("nonexistent")); } @Test(expected = IllegalArgumentException.class) public void require_that_requesting_session_for_empty_application_throws_exception() throws Exception { - curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz:bim"); + ApplicationId baz = createApplicationId("baz"); + // No data in node + curatorFramework.create().creatingParentsIfNeeded() + .forPath(Tenants.getApplicationsPath(tenantName).append(baz.serializedForm()).getAbsolute()); TenantApplications repo = createZKAppRepo(); - repo.getSessionIdForApplication(new ApplicationId.Builder() - .tenant("tenant") - .applicationName("foo").instanceName("bim").build()); + repo.getSessionIdForApplication(baz); } @Test public void require_that_application_ids_can_be_written() throws Exception { TenantApplications repo = createZKAppRepo(); - repo.createPutApplicationTransaction(createAppplicationId("myapp"), 3l).commit(); - String path = "/mytenant:myapp:myinst"; + ApplicationId myapp = createApplicationId("myapp"); + repo.createPutApplicationTransaction(myapp, 3l).commit(); + String path = Tenants.getApplicationsPath(tenantName).append(myapp.serializedForm()).getAbsolute(); assertTrue(curatorFramework.checkExists().forPath(path) != null); assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("3")); - repo.createPutApplicationTransaction(createAppplicationId("myapp"), 5l).commit(); + repo.createPutApplicationTransaction(myapp, 5l).commit(); assertTrue(curatorFramework.checkExists().forPath(path) != null); assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("5")); } @@ -79,8 +80,8 @@ public class TenantApplicationsTest extends TestWithCurator { @Test public void require_that_application_ids_can_be_deleted() throws Exception { TenantApplications repo = createZKAppRepo(); - ApplicationId id1 = createAppplicationId("myapp"); - ApplicationId id2 = createAppplicationId("myapp2"); + ApplicationId id1 = createApplicationId("myapp"); + ApplicationId id2 = createApplicationId("myapp2"); repo.createPutApplicationTransaction(id1, 1).commit(); repo.createPutApplicationTransaction(id2, 1).commit(); assertThat(repo.listApplications().size(), is(2)); @@ -95,8 +96,8 @@ public class TenantApplicationsTest extends TestWithCurator { TenantApplications zkRepo = createZKAppRepo(); TenantApplications memRepo = new MemoryTenantApplications(); for (TenantApplications repo : Arrays.asList(zkRepo, memRepo)) { - ApplicationId id1 = createAppplicationId("myapp"); - ApplicationId id2 = createAppplicationId("myapp2"); + ApplicationId id1 = createApplicationId("myapp"); + ApplicationId id2 = createApplicationId("myapp2"); repo.createPutApplicationTransaction(id1, 4).commit(); repo.createPutApplicationTransaction(id2, 5).commit(); List<ApplicationId> lst = repo.listApplications(); @@ -122,21 +123,19 @@ public class TenantApplicationsTest extends TestWithCurator { @Test public void require_that_reload_handler_is_called_when_apps_are_removed() throws Exception { - curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:test:baz", Utf8.toAsciiBytes(3)); - curatorFramework.create().creatingParentsIfNeeded().forPath("/bar:dev:bim", Utf8.toAsciiBytes(4)); + ApplicationId foo = createApplicationId("foo"); + writeApplicationData(foo, 3L); + writeApplicationData(createApplicationId("bar"), 4L); MockReloadHandler reloadHandler = new MockReloadHandler(); TenantApplications repo = createZKAppRepo(reloadHandler); assertNull(reloadHandler.lastRemoved); - repo.deleteApplication(new ApplicationId.Builder() - .tenant("foo") - .applicationName("test").instanceName("baz").build()) - .commit(); + repo.deleteApplication(foo).commit(); long endTime = System.currentTimeMillis() + 60_000; while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { Thread.sleep(100); } assertNotNull(reloadHandler.lastRemoved); - assertThat(reloadHandler.lastRemoved.serializedForm(), is("foo:test:baz")); + assertThat(reloadHandler.lastRemoved.serializedForm(), is(foo.serializedForm())); } private TenantApplications createZKAppRepo() { @@ -144,12 +143,26 @@ public class TenantApplicationsTest extends TestWithCurator { } private TenantApplications createZKAppRepo(MockReloadHandler reloadHandler) { - return ZKTenantApplications.create(curator, Path.createRoot(), reloadHandler, TenantName.from("mytenant")); + return ZKTenantApplications.create(curator, reloadHandler, tenantName); } - private static ApplicationId createAppplicationId(String name) { + private static ApplicationId createApplicationId(String name) { return new ApplicationId.Builder() - .tenant("mytenant") - .applicationName(name).instanceName("myinst").build(); + .tenant(tenantName.value()) + .applicationName(name) + .instanceName("myinst") + .build(); + } + + private void writeApplicationData(ApplicationId applicationId, long sessionId) throws Exception { + writeApplicationData(applicationId.serializedForm(), sessionId); + } + + private void writeApplicationData(String applicationId, long sessionId) throws Exception { + curatorFramework + .create() + .creatingParentsIfNeeded() + .forPath(Tenants.getApplicationsPath(tenantName).append(applicationId).getAbsolute(), + Utf8.toAsciiBytes(sessionId)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 64932700173..d9a0db7e811 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -69,26 +69,19 @@ public class DeployTester { } public DeployTester(String appPath, List<ModelFactory> modelFactories) { - this(appPath, modelFactories, new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir() - .getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir() - .getAbsolutePath())), + this(appPath, modelFactories, + new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(Files.createTempDir().getAbsolutePath()) + .configDefinitionsDir(Files.createTempDir().getAbsolutePath())), Clock.systemUTC()); } public DeployTester(String appPath, ConfigserverConfig configserverConfig) { - this(appPath, - Collections.singletonList(createModelFactory(Clock.systemUTC())), - configserverConfig, - Clock.systemUTC()); + this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC()); } public DeployTester(String appPath, ConfigserverConfig configserverConfig, Clock clock) { - this(appPath, - Collections.singletonList(createModelFactory(clock)), - configserverConfig, - clock); + this(appPath, Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); } public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { @@ -96,24 +89,22 @@ public class DeployTester { } public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { - Metrics metrics = Metrics.createTestMetrics(); - Curator curator = new MockCurator(); this.clock = clock; - TestComponentRegistry componentRegistry = createComponentRegistry(curator, metrics, modelFactories, - configserverConfig, clock); + TestComponentRegistry componentRegistry = createComponentRegistry(new MockCurator(), Metrics.createTestMetrics(), + modelFactories, configserverConfig, clock); try { this.testApp = new File(appPath); - this.tenants = new Tenants(componentRegistry, metrics); + this.tenants = new Tenants(componentRegistry, Collections.emptySet()); } catch (Exception e) { throw new IllegalArgumentException(e); } - applicationRepository = new ApplicationRepository(tenants, - createHostProvisioner(), - clock); + applicationRepository = new ApplicationRepository(tenants, createHostProvisioner(), clock); } - public Tenant tenant() { return tenants.defaultTenant(); } + public Tenant tenant() { + return tenants.defaultTenant(); + } /** Create a model factory for the version of this source*/ public static ModelFactory createModelFactory(Clock clock) { @@ -139,6 +130,7 @@ public class DeployTester { * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public ApplicationId deployApp(String appName, String vespaVersion, Instant now) { + Tenant tenant = tenant(); LocalSession session = tenant.getSessionFactory().createSession(testApp, appName, new TimeoutBudget(clock, Duration.ofSeconds(60))); ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java new file mode 100644 index 00000000000..09260987ac0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -0,0 +1,112 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; +import com.yahoo.net.HostName; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class FileServerTest { + + FileServer fs = new FileServer(new File(".")); + List<File> created = new LinkedList<>(); + + private void createCleanDir(String name) throws IOException{ + File dir = new File(name); + IOUtils.recursiveDeleteDir(dir); + IOUtils.createDirectory(dir.getName()); + File dummy = new File(dir.getName() +"/dummy"); + IOUtils.writeFile(dummy, "test", true); + assertTrue(dummy.delete()); + created.add(dir); + } + + @Test + public void requireThatExistingFileCanbeFound() throws IOException { + createCleanDir("123"); + IOUtils.writeFile("123/f1", "test", true); + assertTrue(fs.hasFile("123")); + cleanup(); + } + + @Test + public void requireThatNonExistingFileCanNotBeFound() throws IOException { + assertFalse(fs.hasFile("12x")); + createCleanDir("12x"); + assertFalse(fs.hasFile("12x")); + cleanup(); + } + + private static class FileReceiver implements FileServer.Receiver { + CompletableFuture<byte []> content; + FileReceiver(CompletableFuture<byte []> content) { + this.content = content; + } + @Override + public void receive(FileReference reference, String filename, byte[] content, FileServer.ReplayStatus status) { + this.content.complete(content); + } + } + + @Test + public void requireThatWeCanReplayFile() throws IOException, InterruptedException, ExecutionException { + createCleanDir("12y"); + IOUtils.writeFile("12y/f1", "dummy-data", true); + CompletableFuture<byte []> content = new CompletableFuture<>(); + fs.startFileServing("12y", new FileReceiver(content)); + assertEquals(new String(content.get()), "dummy-data"); + cleanup(); + } + + @Test + public void requireThatDifferentNumberOfConfigServersWork() throws IOException { + // Empty connection pool in tests etc. + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); + FileServer fileServer = new FileServer(new ConfigserverConfig(builder)); + assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); + + // Empty connection pool when only one server, no use in downloading from yourself + List<ConfigserverConfig.Zookeeperserver.Builder> servers = new ArrayList<>(); + ConfigserverConfig.Zookeeperserver.Builder serverBuilder = new ConfigserverConfig.Zookeeperserver.Builder(); + serverBuilder.hostname(HostName.getLocalhost()); + serverBuilder.port(123456); + servers.add(serverBuilder); + builder.zookeeperserver(servers); + fileServer = new FileServer(new ConfigserverConfig(builder)); + assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); + + // connection pool of size 1 when 2 servers + ConfigserverConfig.Zookeeperserver.Builder serverBuilder2 = new ConfigserverConfig.Zookeeperserver.Builder(); + serverBuilder2.hostname("bar"); + serverBuilder2.port(123456); + servers.add(serverBuilder2); + builder.zookeeperserver(servers); + fileServer = new FileServer(new ConfigserverConfig(builder)); + assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); + } + + private void cleanup() { + created.forEach((file) -> IOUtils.recursiveDeleteDir(file)); + created.clear(); + } + + @Override + protected void finalize() throws Throwable { + super.finalize(); + cleanup(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStreamTest.java index 5d23f1a4556..ddd29f96695 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStreamTest.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; +import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -10,16 +10,19 @@ import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author lulf @@ -37,6 +40,7 @@ public class CompressedApplicationInputStreamTest { File app = new File("src/test/resources/deploy/validapp"); writeFileToTar(taos, new File(app, "services.xml")); writeFileToTar(taos, new File(app, "hosts.xml")); + writeFileToTar(taos, new File(app, "deployment.xml")); taos.close(); return outFile; } @@ -55,14 +59,8 @@ public class CompressedApplicationInputStreamTest { void assertTestApp(File outApp) { String [] files = outApp.list(); - assertThat(files.length, is(2)); - if ("hosts.xml".equals(files[0])) { - assertThat(files[1], is("services.xml")); - } else if ("hosts.xml".equals(files[1])) { - assertThat(files[0], is("services.xml")); - } else { - fail("Both services.xml and hosts.xml should be contained in the unpacked application"); - } + assertThat(files.length, is(3)); + assertThat(Arrays.asList(files), containsInAnyOrder(ImmutableList.of(is("hosts.xml"), is("services.xml"), is("deployment.xml")))); } @Test @@ -88,6 +86,10 @@ public class CompressedApplicationInputStreamTest { archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); ByteStreams.copy(new FileInputStream(file), archiveOutputStream); archiveOutputStream.closeArchiveEntry(); + file = new File(app, "deployment.xml"); + archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); + ByteStreams.copy(new FileInputStream(file), archiveOutputStream); + archiveOutputStream.closeArchiveEntry(); archiveOutputStream.close(); @@ -134,9 +136,10 @@ public class CompressedApplicationInputStreamTest { new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); File outApp = unpacked.decompress(); List<File> files = Arrays.asList(outApp.listFiles()); - assertThat(files.size(), is(4)); + assertThat(files.size(), is(5)); assertTrue(files.contains(new File(outApp, "services.xml"))); assertTrue(files.contains(new File(outApp, "hosts.xml"))); + assertTrue(files.contains(new File(outApp, "deployment.xml"))); assertTrue(files.contains(new File(outApp, "searchdefinitions"))); assertTrue(files.contains(new File(outApp, "external"))); File sd = files.get(files.indexOf(new File(outApp, "searchdefinitions"))); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 892c821950e..5552758a0a6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -292,7 +292,7 @@ public class ApplicationHandlerTest { Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) .build(); - Tenants tenants = new Tenants(componentRegistry, Metrics.createTestMetrics()); // Creates the application path element in zk + Tenants tenants = new Tenants(componentRegistry); // Creates the application path element in zk tenants.addTenant(tenantName); Tenant tenant = tenants.getTenant(tenantName); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 6b25772f85d..9d04b7e982d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -19,10 +19,8 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.path.Path; import com.yahoo.slime.JsonFormat; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.config.server.SuperModelGenerationCounter; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; @@ -45,6 +43,7 @@ import com.yahoo.vespa.config.server.session.SessionContext; import com.yahoo.vespa.config.server.session.SessionFactory; import com.yahoo.vespa.config.server.session.SessionTest; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -83,7 +82,6 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { private Curator curator; private RemoteSessionRepo remoteSessionRepo; private LocalSessionRepo localRepo; - private PathProvider pathProvider; private TenantApplications applicationRepo; private MockProvisioner hostProvisioner; @@ -95,7 +93,6 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { configCurator = ConfigCurator.create(curator); localRepo = new LocalSessionRepo(Clock.systemUTC()); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; - pathProvider = new PathProvider(Path.createRoot()); hostProvisioner = new MockProvisioner(); } @@ -213,7 +210,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { private RemoteSession createRemoteSession(long sessionId, Session.Status status, SessionZooKeeperClient zkClient, Clock clock) throws IOException { zkClient.writeStatus(status); - ZooKeeperClient zkC = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, pathProvider.getSessionDirs().append(String.valueOf(sessionId))); + ZooKeeperClient zkC = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, Tenants.getSessionsPath(tenant).append(String.valueOf(sessionId))); VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); zkC.write(Collections.singletonMap(modelFactory.getVersion(), new MockFileRegistry())); zkC.write(AllocatedHosts.withHosts(Collections.emptySet())); @@ -318,7 +315,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { } ActivateRequest invoke(boolean createLocalSession) throws Exception { - SessionZooKeeperClient zkClient = new MockSessionZKClient(curator, pathProvider.getSessionDirs().append(String.valueOf(sessionId)), + SessionZooKeeperClient zkClient = new MockSessionZKClient(curator, tenant, sessionId, Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); session = createRemoteSession(sessionId, initialStatus, zkClient, clock); if (createLocalSession) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 7fe7b350734..310342e81f1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; import com.yahoo.vespa.config.server.application.TenantApplications; @@ -15,24 +15,33 @@ import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.session.*; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.SessionFactory; import com.yahoo.vespa.config.server.tenant.Tenants; -import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.time.Clock; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import static com.yahoo.jdisc.Response.Status.*; +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.OK; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -import static com.yahoo.jdisc.http.HttpRequest.Method.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author hmusum @@ -168,7 +177,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { assertTrue(applicationPackage.exists()); final File[] files = applicationPackage.listFiles(); assertNotNull(files); - assertThat(files.length, is(2)); + assertThat(files.length, is(3)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 428cd16508f..7900a67bddd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -20,7 +20,6 @@ import com.yahoo.slime.JsonDecoder; import com.yahoo.slime.Slime; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -33,6 +32,7 @@ import com.yahoo.vespa.config.server.http.*; import com.yahoo.vespa.config.server.session.*; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import org.apache.commons.lang.NullArgumentException; import org.junit.Before; import org.junit.Test; @@ -149,14 +149,13 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { */ private RemoteSessionRepo fromLocalSessionRepo(LocalSessionRepo localRepo, Clock clock) { RemoteSessionRepo remoteRepo = new RemoteSessionRepo(); - PathProvider pathProvider = new PathProvider(Path.createRoot()); for (LocalSession ls : localRepo.listSessions()) { - zooKeeperClient = new MockSessionZKClient(curator, pathProvider.getSessionDirs().append(String.valueOf(ls.getSessionId()))); + zooKeeperClient = new MockSessionZKClient(curator, tenant, ls.getSessionId()); if (ls.getStatus()!=null) zooKeeperClient.writeStatus(ls.getStatus()); - RemoteSession remSess = new RemoteSession(TenantName.from("default"), ls.getSessionId(), + RemoteSession remSess = new RemoteSession(tenant, ls.getSessionId(), new TestComponentRegistry.Builder().curator(curator).build(), - zooKeeperClient, + zooKeeperClient, clock); remoteRepo.addSession(remSess); } @@ -239,8 +238,8 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_that_preparing_with_multiple_tenants_work() throws Exception { // Need different repos for 'default' tenant as opposed to the 'test' tenant LocalSessionRepo localRepoDefault = new LocalSessionRepo(Clock.systemUTC()); - final TenantName tenantName = TenantName.defaultName(); - addTenant(tenantName, localRepoDefault, new RemoteSessionRepo(), new MockSessionFactory()); + final TenantName defaultTenant = TenantName.defaultName(); + addTenant(defaultTenant, localRepoDefault, new RemoteSessionRepo(), new MockSessionFactory()); addTestTenant(); final SessionHandler handler = createHandler(builder); @@ -248,7 +247,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { // Deploy with default tenant MockSession session = new MockSession(sessionId, null); localRepoDefault.addSession(session); - pathPrefix = "/application/v2/tenant/default/session/"; + pathPrefix = "/application/v2/tenant/" + defaultTenant + "/session/"; HttpResponse response = handler.handle( SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); @@ -317,7 +316,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void test_out_of_capacity_response() throws InterruptedException, IOException { - String message = "No nodes available"; + String message = "Internal error"; SessionThrowingException session = new SessionThrowingException(new OutOfCapacityException(message)); localRepo.addSession(session); HttpResponse response = createHandler() @@ -329,6 +328,19 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { } @Test + public void test_that_nullpointerexception_gives_internal_server_error() throws InterruptedException, IOException { + String message = "No nodes available"; + SessionThrowingException session = new SessionThrowingException(new NullPointerException(message)); + localRepo.addSession(session); + HttpResponse response = createHandler() + .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + assertEquals(500, response.getStatus()); + Slime data = getData(response); + assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR.name())); + assertThat(data.get().field("message").asString(), is(message)); + } + + @Test public void test_application_lock_failure() throws InterruptedException, IOException { String message = "Timed out after waiting PT1M to acquire lock '/provision/v1/locks/foo/bar/default'"; SessionThrowingException session = diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java index d82f62cfc1a..9dbb193ab3d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java @@ -9,7 +9,6 @@ import java.util.concurrent.Executor; import com.yahoo.vespa.config.server.*; import com.yahoo.vespa.config.server.http.SessionResponse; -import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.Tenants; import org.junit.After; import org.junit.Before; @@ -35,7 +34,7 @@ public class TenantTest extends TestWithCurator { } protected Tenants createTenants() throws Exception { - return new Tenants(new TestComponentRegistry.Builder().curator(curator).build(), Metrics.createTestMetrics()); + return new Tenants(new TestComponentRegistry.Builder().curator(curator).build()); } protected Executor testExecutor() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java index 2b1000c2211..16ce605d4d1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java @@ -4,10 +4,9 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; -import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -19,7 +18,7 @@ import java.util.*; /** * Test utility for creating tenants used for testing and setup wiring of tenant stuff. * - * @author lulf + * @author Ulf Lilleengen * @since 5.1 */ public class TestTenantBuilder { @@ -33,7 +32,7 @@ public class TestTenantBuilder { public TenantBuilder createTenant(TenantName tenantName) { MemoryTenantApplications applicationRepo = new MemoryTenantApplications(); - TenantBuilder builder = TenantBuilder.create(componentRegistry, tenantName, Path.createRoot().append(tenantName.value())) + TenantBuilder builder = TenantBuilder.create(componentRegistry, tenantName) .withSessionFactory(new SessionCreateHandlerTest.MockSessionFactory()) .withLocalSessionRepo(new LocalSessionRepo(componentRegistry.getClock())) .withRemoteSessionRepo(new RemoteSessionRepo()) @@ -57,6 +56,6 @@ public class TestTenantBuilder { } } }); - return new Tenants(componentRegistry, Metrics.createTestMetrics(), tenantList); + return new Tenants(componentRegistry, tenantList); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 474b93f6972..df8ed405fe3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -88,8 +88,9 @@ public class LbServicesProducerTest { private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) throws IOException, SAXException { final Zone zone = new Zone(Environment.prod, regionName); - Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder(). - properties(new DeployProperties.Builder().zone(zone).build())); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() + .properties(new DeployProperties.Builder().zone(zone).build()) + .zone(zone)); return getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java index 122deadb841..0126a9e2f29 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java @@ -12,8 +12,6 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.GetConfigContext; -import com.yahoo.vespa.config.server.rpc.DelayedConfigResponses; -import com.yahoo.vespa.config.server.rpc.MockRpc; import org.junit.Test; import java.util.Collections; @@ -58,7 +56,7 @@ public class DelayedConfigResponseTest { DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); assertThat(responses.size(), is(1)); - responses.allDelayedResponses().get(0).cancel(); + responses.allDelayedResponses().get(0).cancelAndRemove(); assertThat(responses.size(), is(0)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index a08514e8afb..4c2a4b56751 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -2,11 +2,13 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.server.GetConfigContext; +import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -37,7 +39,7 @@ public class MockRpc extends RpcServer { public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) { super(createConfig(port), null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker()); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); if (createDefaultTenant) { onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index fa6adb64a8a..12dc584f055 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; @@ -12,6 +13,7 @@ import com.yahoo.net.HostName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -88,7 +90,7 @@ public class TestWithRpc { emptyNodeFlavors(), generationCounter)), Metrics.createTestMetrics(), new HostRegistries(), - hostLivenessTracker); + hostLivenessTracker, new FileServer(FileDistribution.getDefaultFileDBPath())); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); t.start(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 5753b2959f7..3d34d08edeb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -2,10 +2,11 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.path.Path; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.config.server.*; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.TestWithCurator; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.io.IOUtils; @@ -20,13 +21,12 @@ import java.io.File; import java.time.Duration; import java.time.Instant; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; /** - * @author lulf + * @author Ulf Lilleengen * @since 5.1 */ public class LocalSessionRepoTest extends TestWithCurator { @@ -51,10 +51,7 @@ public class LocalSessionRepoTest extends TestWithCurator { } clock = new ManualClock(Instant.ofEpochSecond(1)); LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, - new SessionCounter(globalComponentRegistry.getCurator(), - Path.fromString("counter"), - Path.fromString("sessions")), - Path.createRoot(), + new SessionCounter(globalComponentRegistry.getCurator(), tenantName), new MemoryTenantApplications(), tenantFileSystemDirs, new HostRegistry<>(), tenantName); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index c6099e724bc..b98fa49ac26 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -108,13 +109,15 @@ public class LocalSessionTest { @Test public void require_that_session_can_be_deleted() throws Exception { - LocalSession session = createSession(TenantName.defaultName(), 3); - assertTrue(configCurator.exists("/3")); + TenantName tenantName = TenantName.defaultName(); + LocalSession session = createSession(tenantName, 3); + String sessionNode = Tenants.getSessionsPath(tenantName).append(String.valueOf(3)).getAbsolute(); + assertTrue(configCurator.exists(sessionNode)); assertTrue(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); long gen = superModelGenerationCounter.get(); session.delete(); assertThat(superModelGenerationCounter.get(), is(gen + 1)); - assertFalse(configCurator.exists("/3")); + assertFalse(configCurator.exists(sessionNode)); assertFalse(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); } @@ -155,10 +158,9 @@ public class LocalSessionTest { } private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<AllocatedHosts> allocatedHosts) throws Exception { - Path sessionPath = Path.fromString("/" + sessionId); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, sessionPath, allocatedHosts); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenant, sessionId, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, sessionPath); + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, Tenants.getSessionsPath(tenant).append(String.valueOf(sessionId))); if (allocatedHosts.isPresent()) { zkClient.write(allocatedHosts.get()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java index 62b0ecbada2..a4331216334 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.TenantName; import com.yahoo.transaction.Transaction; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -22,17 +24,17 @@ public class MockSessionZKClient extends SessionZooKeeperClient { private Optional<AllocatedHosts> info = Optional.empty(); private Session.Status sessionStatus; - public MockSessionZKClient(Curator curator, Path sessionPath) { - this(curator, sessionPath, (ApplicationPackage)null); + public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId) { + this(curator, tenantName, sessionId, (ApplicationPackage)null); } - public MockSessionZKClient(Curator curator, Path sessionPath, Optional<AllocatedHosts> allocatedHosts) { - this(curator, sessionPath); + public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, Optional<AllocatedHosts> allocatedHosts) { + this(curator, tenantName, sessionId); this.info = allocatedHosts; } - public MockSessionZKClient(Curator curator, Path sessionPath, ApplicationPackage application) { - super(curator, sessionPath); + public MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, ApplicationPackage application) { + super(curator, Tenants.getSessionsPath(tenantName).append(String.valueOf(sessionId))); this.app = application; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java index 462062ce8a8..878339bd703 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java @@ -2,18 +2,22 @@ package com.yahoo.vespa.config.server.session; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.TestWithCurator; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.Curator; import org.junit.Before; import org.junit.Test; @@ -27,33 +31,35 @@ import java.util.concurrent.TimeUnit; import java.util.function.LongPredicate; /** - * @author lulf + * @author Ulf Lilleengen * @since 5.1 */ public class RemoteSessionRepoTest extends TestWithCurator { + private static final TenantName tenantName = TenantName.defaultName(); + private RemoteSessionRepo remoteSessionRepo; @Before public void setupFacade() throws Exception { - createSession(2l, false); - createSession(3l, false); - curator.create(Path.fromString("/applications")); - curator.create(Path.fromString("/sessions")); - Tenant tenant = TenantBuilder.create(new TestComponentRegistry.Builder().curator(curator).build(), - TenantName.defaultName(), - Path.createRoot()).build(); + Tenant tenant = TenantBuilder.create(new TestComponentRegistry.Builder() + .curator(curator) + .build(), + tenantName) + .build(); this.remoteSessionRepo = tenant.getRemoteSessionRepo(); + curator.create(Tenants.getTenantPath(tenantName).append("/applications")); + curator.create(Tenants.getSessionsPath(tenantName)); + createSession(1l, false); + createSession(2l, false); } private void createSession(long sessionId, boolean wait) { - createSession("", sessionId, wait); + createSession(sessionId, wait, tenantName); } - - private void createSession(String root, long sessionId, boolean wait) { - Path sessionsPath = Path.fromString(root).append("sessions"); - curator.create(sessionsPath); + private void createSession(long sessionId, boolean wait, TenantName tenantName) { + Path sessionsPath = Tenants.getSessionsPath(tenantName); SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); zkc.createNewSession(System.currentTimeMillis(), TimeUnit.MILLISECONDS); if (wait) { @@ -64,27 +70,28 @@ public class RemoteSessionRepoTest extends TestWithCurator { @Test public void testInitialize() { + assertSessionExists(1l); assertSessionExists(2l); - assertSessionExists(3l); } @Test public void testCreateSession() throws Exception { - createSession(0l, true); - assertSessionExists(0l); + createSession(3l, true); + assertSessionExists(3l); } @Test public void testSessionStateChange() throws Exception { - Path session = Path.fromString("/sessions/0"); - createSession(0l, true); - assertSessionStatus(0l, Session.Status.NEW); - assertStatusChange(0l, Session.Status.PREPARE); - assertStatusChange(0l, Session.Status.ACTIVATE); + long sessionId = 3L; + createSession(sessionId, true); + assertSessionStatus(sessionId, Session.Status.NEW); + assertStatusChange(sessionId, Session.Status.PREPARE); + assertStatusChange(sessionId, Session.Status.ACTIVATE); + Path session = Tenants.getSessionsPath(tenantName).append("" + sessionId); curator.delete(session); - assertSessionRemoved(0l); - assertNull(remoteSessionRepo.getSession(0l)); + assertSessionRemoved(sessionId); + assertNull(remoteSessionRepo.getSession(sessionId)); } // If reading a session throws an exception it should be handled and not prevent other applications @@ -93,25 +100,25 @@ public class RemoteSessionRepoTest extends TestWithCurator { // throw an exception). @Test public void testBadApplicationRepoOnActivate() throws Exception { + long sessionId = 3L; TenantApplications applicationRepo = new FailingTenantApplications(); - curator.framework().create().forPath("/mytenant"); - Tenant tenant = TenantBuilder.create(new TestComponentRegistry.Builder().curator(curator).build(), - TenantName.from("mytenant"), - Path.fromString("mytenant")) + TenantName mytenant = TenantName.from("mytenant"); + Tenant tenant = TenantBuilder.create(new TestComponentRegistry.Builder().curator(curator).build(), mytenant) .withApplicationRepo(applicationRepo) .build(); + curator.create(Tenants.getSessionsPath(mytenant)); remoteSessionRepo = tenant.getRemoteSessionRepo(); assertThat(remoteSessionRepo.listSessions().size(), is(0)); - createSession("/mytenant", 2l, true); + createSession(sessionId, true, mytenant); assertThat(remoteSessionRepo.listSessions().size(), is(1)); } private void assertStatusChange(long sessionId, Session.Status status) throws Exception { - Path statePath = Path.fromString("/sessions/" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + Path statePath = Tenants.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); curator.create(statePath); curatorFramework.setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); System.out.println("Setting status " + status + " for " + sessionId); - assertSessionStatus(0l, status); + assertSessionStatus(sessionId, status); } private void assertSessionRemoved(long sessionId) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java index 44f304847ba..9598a9262f0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -11,10 +11,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Version; -import com.yahoo.path.Path; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.curator.mock.MockCurator; @@ -49,13 +47,13 @@ import static org.junit.Assert.assertTrue; */ public class RemoteSessionTest { + private static final TenantName tenantName = TenantName.from("default"); + private Curator curator; - private PathProvider pathProvider; @Before public void setupTest() throws Exception { curator = new MockCurator(); - pathProvider = new PathProvider(Path.createRoot()); } @Test @@ -180,7 +178,7 @@ public class RemoteSessionTest { okFactory.vespaVersion = Version.fromIntValues(2, 0, 0); okFactory.throwOnLoad = false; - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(3), application); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, 3, application); RemoteSession session = createSession(3, zkc, Arrays.asList(okFactory, failingFactory), failingFactory.clock()); session.loadPrepared(); @@ -189,7 +187,7 @@ public class RemoteSessionTest { @Test public void require_that_session_status_is_updated() throws IOException, SAXException { - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(3)); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, 3); RemoteSession session = createSession(3, zkc, Clock.systemUTC()); assertThat(session.getStatus(), is(Session.Status.NEW)); zkc.writeStatus(Session.Status.PREPARE); @@ -203,7 +201,7 @@ public class RemoteSessionTest { MockModelFactory mockModelFactory = new MockModelFactory(); try { int sessionId = 3; - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(sessionId)); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId); createSession(sessionId, zkc, Collections.singletonList(mockModelFactory), permanentApp, mockModelFactory.clock()).ensureApplicationLoaded(); } catch (Exception e) { e.printStackTrace(); @@ -220,7 +218,7 @@ public class RemoteSessionTest { return createSession(sessionId, zkc, Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())), clock); } private RemoteSession createSession(long sessionId, List<ModelFactory> modelFactories, Clock clock) { - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(sessionId)); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId); return createSession(sessionId, zkc, modelFactories, clock); } @@ -238,7 +236,8 @@ public class RemoteSessionTest { if (permanentApplicationPackage.isPresent()) registryBuilder.permanentApplicationPackage(permanentApplicationPackage.get()); - return new RemoteSession(TenantName.from("default"), sessionId, + + return new RemoteSession(tenantName, sessionId, registryBuilder.build(), zkc, clock); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 5dc529e3381..2069ae48d76 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -126,7 +126,6 @@ public class SessionPreparerTest extends TestWithCurator { new PrepareParams.Builder().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()).build(), Optional.empty(), tenantPath, Instant.now()); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(0)); - assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().limitSendingOfDeployedFilesToCalled, is(0)); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(0)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java index 1bb25bc37db..01cb90721f3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Test; @@ -34,10 +32,10 @@ public class SessionRepoTest { } private class TestSession extends Session { - public TestSession(long sessionId) { - super(TenantName.from("default"), + TestSession(long sessionId) { + super(TenantName.defaultName(), sessionId, - new MockSessionZKClient(new MockCurator(), new PathProvider(Path.createRoot()).getSessionDir(sessionId))); + new MockSessionZKClient(new MockCurator(), TenantName.defaultName(), sessionId)); } } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 91cf6e79165..dc6268c5a25 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.GetConfigRequest; @@ -20,7 +19,6 @@ import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.config.server.ReloadListener; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; @@ -88,7 +86,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { private void feedApp(File appDir, long sessionId, ApplicationId appId) throws IOException { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, - new PathProvider(Path.createRoot()).getSessionDir(sessionId), + Tenants.getSessionsPath(tenant).append(String.valueOf(sessionId)), new TestConfigDefinitionRepo(), "", Optional.empty()); zkc.writeApplicationId(appId); @@ -104,7 +102,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { private ApplicationSet reloadConfig(long id, String application, Clock clock) { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, - new PathProvider(Path.createRoot()).getSessionDir(id), + Tenants.getSessionsPath(tenant).append(String.valueOf(id)), new TestConfigDefinitionRepo(), "", Optional.empty()); zkc.writeApplicationId(new ApplicationId.Builder().tenant(tenant).applicationName(application).build()); @@ -187,7 +185,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { public void testResolveForAppId() { long id = 1l; SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, - new PathProvider(Path.createRoot()).getSessionDir(id), + Tenants.getSessionsPath(tenant).append(String.valueOf(id)), new TestConfigDefinitionRepo(), "", Optional.empty()); ApplicationId appId = new ApplicationId.Builder() @@ -231,7 +229,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { feedApp(appDir, sessionId, appId); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, new PathProvider(Path.createRoot()).getSessionDir(sessionId)); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, Tenants.getSessionsPath(tenant).append(String.valueOf(sessionId))); zkc.writeApplicationId(appId); RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc, Clock.systemUTC()); server.reloadConfig(session.ensureApplicationLoaded()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java index d1724986d5e..e650997b7e0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TestWithCurator; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.model.VespaModel; import org.junit.After; import org.junit.Before; @@ -20,6 +19,7 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -46,7 +46,7 @@ public class TenantsTestCase extends TestWithCurator { listener = (TenantRequestHandlerTest.MockReloadListener)globalComponentRegistry.getReloadListener(); tenantListener = (MockTenantListener)globalComponentRegistry.getTenantListener(); tenantListener.tenantsLoaded = false; - tenants = new Tenants(globalComponentRegistry, Metrics.createTestMetrics()); + tenants = new Tenants(globalComponentRegistry); assertTrue(tenantListener.tenantsLoaded); tenants.addTenant(tenant1); tenants.addTenant(tenant2); @@ -105,19 +105,28 @@ public class TenantsTestCase extends TestWithCurator { } @Test - public void testRemove() throws Exception { + public void testDelete() throws Exception { assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenants.tenantZkPath(tenant1))); tenants.deleteTenant(tenant1); assertFalse(tenants.getAllTenantNames().contains(tenant1)); } + + @Test(expected = IllegalArgumentException.class) + public void testDeleteOfDefaultTenant() { + try { + assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenants.tenantZkPath(TenantName.defaultName()))); + } catch (Exception e) { + fail("default tenant does not exist"); + } + tenants.deleteTenant(TenantName.defaultName()); + } @Test public void testTenantsChanged() throws Exception { tenants.close(); // close the Tenants instance created in setupSession, we do not want to use one with a PatchChildrenCache listener - tenants = new Tenants(globalComponentRegistry, Metrics.createTestMetrics(), new ArrayList<>()); + tenants = new Tenants(globalComponentRegistry, new ArrayList<>()); TenantName defaultTenant = TenantName.defaultName(); tenants.addTenant(tenant2); - tenants.addTenant(defaultTenant); tenants.createTenants(); Set<TenantName> allTenants = tenants.getAllTenantNames(); assertTrue(allTenants.contains(tenant2)); @@ -125,12 +134,10 @@ public class TenantsTestCase extends TestWithCurator { assertTrue(allTenants.contains(defaultTenant)); tenants.deleteTenant(tenant1); tenants.deleteTenant(tenant2); - tenants.deleteTenant(defaultTenant); tenants.createTenants(); allTenants = tenants.getAllTenantNames(); assertFalse(allTenants.contains(tenant1)); assertFalse(allTenants.contains(tenant2)); - assertFalse(allTenants.contains(defaultTenant)); TenantName foo = TenantName.from("foo"); TenantName bar = TenantName.from("bar"); tenants.addTenant(tenant2); @@ -145,25 +152,24 @@ public class TenantsTestCase extends TestWithCurator { @Test public void testTenantWatching() throws Exception { - TestComponentRegistry reg = new TestComponentRegistry.Builder().curator(curator).build(); - Tenants t = new Tenants(reg, Metrics.createTestMetrics()); + TenantName newTenant = TenantName.from("newTenant"); + List<TenantName> expectedTenants = Arrays.asList(TenantName.defaultName(), newTenant); try { - assertTrue(t.getAllTenantNames().contains(TenantName.defaultName())); - reg.getCurator().framework().create().forPath(tenants.tenantZkPath(TenantName.from("newTenant"))); + tenants.addTenant(newTenant); // Poll for the watcher to pick up the tenant from zk, and add it int tries=0; while(true) { - if (tries > 500) fail("Didn't react on watch"); - if (t.getAllTenantNames().contains(TenantName.from("newTenant"))) { - return; + if (tries > 5000) fail("Didn't react on watch"); + if (tenants.getAllTenantNames().containsAll(expectedTenants)) { + break; } tries++; - Thread.sleep(100); + Thread.sleep(10); } } finally { - t.close(); + assertTrue(tenants.getAllTenantNames().containsAll(expectedTenants)); + tenants.close(); } } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestWithTenant.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestWithTenant.java index c11480f5335..4573f785842 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestWithTenant.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestWithTenant.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TestWithCurator; -import com.yahoo.vespa.config.server.monitoring.Metrics; import org.junit.Before; /** @@ -19,8 +18,7 @@ public class TestWithTenant extends TestWithCurator { @Before public void setupTenant() throws Exception { - final Metrics metrics = Metrics.createTestMetrics(); - tenants = new Tenants(new TestComponentRegistry.Builder().curator(curator).metrics(metrics).build(), metrics); + tenants = new Tenants(new TestComponentRegistry.Builder().curator(curator).build()); tenant = tenants.defaultTenant(); } diff --git a/configserver/src/test/resources/deploy/advancedapp/deployment.xml b/configserver/src/test/resources/deploy/advancedapp/deployment.xml new file mode 100644 index 00000000000..fa1d1388e67 --- /dev/null +++ b/configserver/src/test/resources/deploy/advancedapp/deployment.xml @@ -0,0 +1 @@ +<deployment version='1.0'/>
\ No newline at end of file diff --git a/configserver/src/test/resources/deploy/app/deployment.xml b/configserver/src/test/resources/deploy/app/deployment.xml new file mode 100644 index 00000000000..fa1d1388e67 --- /dev/null +++ b/configserver/src/test/resources/deploy/app/deployment.xml @@ -0,0 +1 @@ +<deployment version='1.0'/>
\ No newline at end of file diff --git a/configserver/src/test/resources/deploy/validapp/deployment.xml b/configserver/src/test/resources/deploy/validapp/deployment.xml new file mode 100644 index 00000000000..fa1d1388e67 --- /dev/null +++ b/configserver/src/test/resources/deploy/validapp/deployment.xml @@ -0,0 +1 @@ +<deployment version='1.0'/>
\ No newline at end of file diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp index f558503771d..6a82426fe27 100644 --- a/configutil/src/lib/configstatus.cpp +++ b/configutil/src/lib/configstatus.cpp @@ -165,7 +165,8 @@ ConfigStatus::action() if (!upToDate) { if (svc.type == "searchnode" || svc.type == "filedistributorservice" || - svc.type == "topleveldispatch") + svc.type == "topleveldispatch" || + svc.type == "logd") { std::cerr << "[generation not up-to-date ignored]" << std::endl; } else { diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index 38c341a6a3e..9120c747293 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -1,20 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; +import com.yahoo.collections.ListMap; +import org.apache.commons.lang.builder.ReflectionToStringBuilder; + import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; +import java.security.Principal; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -import com.yahoo.collections.ListMap; -import org.apache.commons.lang.builder.ReflectionToStringBuilder; import static java.util.stream.Collectors.toMap; @@ -32,6 +31,7 @@ import static java.util.stream.Collectors.toMap; * * @author tonytv * @author bakksjo + * @author bjorncs */ public class AccessLogEntry { public enum CookieType { @@ -91,6 +91,9 @@ public class AccessLogEntry { private String zDataIncrementSlotByOneRequest; private String hostString; private int statusCode; + private String scheme; + private int localPort; + private Principal principal; private ListMap<String,String> keyValues=null; @@ -682,6 +685,45 @@ public class AccessLogEntry { } } + public String getScheme() { + synchronized (monitor) { + return scheme; + } + } + + public void setScheme(String scheme) { + synchronized (monitor) { + requireNull(this.scheme); + this.scheme = scheme; + } + } + + public int getLocalPort() { + synchronized (monitor) { + return localPort; + } + } + + public void setLocalPort(int localPort) { + synchronized (monitor) { + requireZero(this.localPort); + this.localPort = localPort; + } + } + + public Principal getUserPrincipal() { + synchronized (monitor) { + return principal; + } + } + + public void setUserPrincipal(Principal principal) { + synchronized (monitor) { + requireNull(this.principal); + this.principal = principal; + } + } + @Override public String toString() { synchronized (monitor) { @@ -708,4 +750,5 @@ public class AccessLogEntry { throw new IllegalStateException("Attempt to overwrite field that has been assigned. Value: " + value); } } + } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java index 450369d998f..cca8da2e936 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; import java.net.URI; +import java.security.Principal; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Objects; @@ -47,6 +49,8 @@ public class JSONFormatter { generator.writeStartObject(); generator.writeStringField("ip", accessLogEntry.getIpV4Address()); generator.writeNumberField("time", toTimestampInSeconds(accessLogEntry.getTimeStampMillis())); + generator.writeStringField("time-iso8601", + Instant.ofEpochMilli(accessLogEntry.getTimeStampMillis()).toString()); generator.writeNumberField("duration", durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis())); generator.writeNumberField("responsesize", accessLogEntry.getReturnedContentSize()); @@ -56,6 +60,16 @@ public class JSONFormatter { generator.writeStringField("version", accessLogEntry.getHttpVersion()); generator.writeStringField("agent", accessLogEntry.getUserAgent()); generator.writeStringField("host", accessLogEntry.getHostString()); + generator.writeStringField("scheme", accessLogEntry.getScheme()); + generator.writeNumberField("localport", accessLogEntry.getLocalPort()); + + Principal principal = accessLogEntry.getUserPrincipal(); + if (principal != null) { + generator.writeObjectFieldStart("user-principal"); + generator.writeStringField("name", principal.getName()); + generator.writeStringField("type", principal.getClass().getName()); + generator.writeEndObject(); + } // Only add remote address/port fields if relevant if (remoteAddressDiffers(accessLogEntry.getIpV4Address(), accessLogEntry.getRemoteAddress())) { diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java index 2d092d5a205..7f81a3568dd 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java @@ -40,6 +40,7 @@ public class JSONLogTestCase extends junit.framework.TestCase { String expectedOutput = "{\"ip\":\"152.200.54.243\"," + "\"time\":920880005.023," + + "\"time-iso8601\":\"1999-03-08T08:00:05.023Z\"," + "\"duration\":0.122," + "\"responsesize\":9875," + "\"code\":200," + @@ -48,6 +49,8 @@ public class JSONLogTestCase extends junit.framework.TestCase { "\"version\":\"HTTP/1.1\"," + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + "\"search\":{" + "\"totalhits\":1234," + "\"hits\":0" + @@ -66,6 +69,7 @@ public class JSONLogTestCase extends junit.framework.TestCase { String expectedOutput = "{\"ip\":\"152.200.54.243\"," + "\"time\":920880005.023," + + "\"time-iso8601\":\"1999-03-08T08:00:05.023Z\"," + "\"duration\":0.122," + "\"responsesize\":9875," + "\"code\":200," + @@ -74,6 +78,8 @@ public class JSONLogTestCase extends junit.framework.TestCase { "\"version\":\"HTTP/1.1\"," + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + "\"search\":{" + "\"totalhits\":1234," + "\"hits\":0" + @@ -96,6 +102,7 @@ public class JSONLogTestCase extends junit.framework.TestCase { String expectedOutput = "{\"ip\":\"152.200.54.243\"," + "\"time\":920880005.023," + + "\"time-iso8601\":\"1999-03-08T08:00:05.023Z\"," + "\"duration\":0.122," + "\"responsesize\":9875," + "\"code\":200," + @@ -104,6 +111,8 @@ public class JSONLogTestCase extends junit.framework.TestCase { "\"version\":\"HTTP/1.1\"," + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + "\"remoteaddr\":\"FE80:0000:0000:0000:0202:B3FF:FE1E:8329\"," + "\"search\":{" + "\"totalhits\":1234," + @@ -119,6 +128,7 @@ public class JSONLogTestCase extends junit.framework.TestCase { expectedOutput = "{\"ip\":\"152.200.54.243\"," + "\"time\":920880005.023," + + "\"time-iso8601\":\"1999-03-08T08:00:05.023Z\"," + "\"duration\":0.122," + "\"responsesize\":9875," + "\"code\":200," + @@ -127,6 +137,8 @@ public class JSONLogTestCase extends junit.framework.TestCase { "\"version\":\"HTTP/1.1\"," + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + "\"remoteaddr\":\"FE80:0000:0000:0000:0202:B3FF:FE1E:8329\"," + "\"remoteport\":1234," + "\"search\":{" + @@ -163,6 +175,7 @@ public class JSONLogTestCase extends junit.framework.TestCase { String expectedOutput = "{\"ip\":\"152.200.54.243\"," + "\"time\":920880005.023," + + "\"time-iso8601\":\"1999-03-08T08:00:05.023Z\"," + "\"duration\":0.122," + "\"responsesize\":9875," + "\"code\":200," + @@ -171,6 +184,8 @@ public class JSONLogTestCase extends junit.framework.TestCase { "\"version\":\"HTTP/1.1\"," + "\"agent\":\"Mozilla/4.05 [en] (Win95; I; \\\"Best Browser Ever\\\")\"," + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + "\"search\":{" + "\"totalhits\":1234," + "\"hits\":0" + diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 7142449f3d4..f6e1c1d6d5a 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -2,10 +2,13 @@ package com.yahoo.container.jdisc; import com.google.inject.Inject; +import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.jdisc.application.UriPattern; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.BufferedContentChannel; import com.yahoo.jdisc.handler.ContentChannel; @@ -22,6 +25,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.Collections.singletonMap; import javax.annotation.concurrent.GuardedBy; @@ -75,6 +79,21 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { this.allowAsyncResponse = allowAsyncResponse; } + private Map<String, Metric.Context> handlerContexts = new CopyOnWriteHashMap<>(); + private Metric.Context contextFor(BindingMatch match) { + if (match == null) return null; + UriPattern matched = match.matched(); + if (matched == null) return null; + String name = matched.toString(); + Metric.Context context = handlerContexts.get(name); + if (context == null) { + Map<String, String> dimensions = singletonMap("handler", name); + context = this.metric.createContext(dimensions); + handlerContexts.put(name, context); + } + return context; + } + /** * Handles a request by assigning a worker thread to it. * @@ -82,6 +101,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { */ @Override public final ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { + metric.add("container.handled.requests", 1, contextFor(request.getBindingMatch())); if (request.getTimeout(TimeUnit.SECONDS) == null) { Duration timeout = getTimeout(); if (timeout != null) { @@ -173,7 +193,10 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { @Override public ContentChannel handleResponse(Response response) { if ( tryHasResponded()) throw new IllegalStateException("Response already handled"); - return responseHandler.handleResponse(response); + ContentChannel cc = responseHandler.handleResponse(response); + long millis = request.timeElapsed(TimeUnit.MILLISECONDS); + metric.set("container.handled.latency", millis, contextFor(request.getBindingMatch())); + return cc; } private boolean tryHasResponded() { diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 43c4b5c33d4..1b0a7f9e958 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -72,6 +72,8 @@ <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.core:jackson-databind:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> + <!-- Use version range for jax deps, because jersey and junit affect the versions. --> <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 8eb8cab1677..d02ec233d96 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -81,12 +81,9 @@ <groupId>org.apache.felix</groupId> <artifactId>org.apache.felix.main</artifactId> </dependency> - <!-- This version is exported by jdisc via jcl-over-slf4j, so should be the one used by customer bundles --> - <!-- Set explicitly here because org.apache.httpcomponents 4.3 wants to pull in 1.1.3 instead. --> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> - <version>1.1.1</version> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/container-disc/pom.xml b/container-disc/pom.xml index bd8a3340622..ad86b3ffcff 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -20,6 +20,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -184,7 +190,7 @@ defaults-jar-with-dependencies.jar, component-jar-with-dependencies.jar, zkfacade-jar-with-dependencies.jar, - <!-- jersey2 --> + <!-- Jersey 2 + Jackson 2 --> aopalliance-repackaged-${hk2.version}.jar, hk2-api-${hk2.version}.jar, hk2-locator-${hk2.version}.jar, @@ -193,6 +199,7 @@ jackson-core-${jackson2.version}.jar, jackson-databind-${jackson2.version}.jar, jackson-datatype-jdk8-${jackson2.version}.jar, + jackson-datatype-jsr310-${jackson2.version}.jar, jackson-jaxrs-base-${jackson2.version}.jar, jackson-jaxrs-json-provider-${jackson2.version}.jar, jackson-module-jaxb-annotations-${jackson2.version}.jar, diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerThreadFactory.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerThreadFactory.java index 379116a5d94..50798a82b60 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerThreadFactory.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ContainerThreadFactory.java @@ -8,7 +8,7 @@ import com.yahoo.jdisc.application.MetricConsumer; import java.util.concurrent.ThreadFactory; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ public class ContainerThreadFactory implements ThreadFactory { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index 19e04e0ae01..033b396bc9b 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -5,9 +5,7 @@ package com.yahoo.container.jdisc.athenz; * @author mortent */ public interface AthenzIdentityProvider { - - String getNToken(); - String getX509Cert(); - String domain(); - String service(); + String getNToken() throws AthenzIdentityProviderException; + String getDomain(); + String getService(); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java new file mode 100644 index 00000000000..fd5839bfc45 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.athenz; + +/** + * @author bjorncs + */ +public class AthenzIdentityProviderException extends RuntimeException { + + public AthenzIdentityProviderException(String message) { + super(message); + } + + public AthenzIdentityProviderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentials.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentials.java new file mode 100644 index 00000000000..790a7c54333 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentials.java @@ -0,0 +1,51 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.athenz.impl; + +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; + +/** + * @author bjorncs + */ +class AthenzCredentials { + + private final String nToken; + private final X509Certificate certificate; + private final KeyPair keyPair; + private final SignedIdentityDocument identityDocument; + private final Instant createdAt; + + AthenzCredentials(String nToken, + X509Certificate certificate, + KeyPair keyPair, + SignedIdentityDocument identityDocument, + Instant createdAt) { + this.nToken = nToken; + this.certificate = certificate; + this.keyPair = keyPair; + this.identityDocument = identityDocument; + this.createdAt = createdAt; + } + + String getNToken() { + return nToken; + } + + X509Certificate getCertificate() { + return certificate; + } + + KeyPair getKeyPair() { + return keyPair; + } + + SignedIdentityDocument getIdentityDocument() { + return identityDocument; + } + + Instant getCreatedAt() { + return createdAt; + } + +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentialsService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentialsService.java new file mode 100644 index 00000000000..5786eb9e398 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzCredentialsService.java @@ -0,0 +1,93 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.athenz.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.core.identity.IdentityConfig; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Clock; + +/** + * @author bjorncs + */ +class AthenzCredentialsService { + + private static final ObjectMapper mapper = new ObjectMapper(); + + private final IdentityConfig identityConfig; + private final IdentityDocumentService identityDocumentService; + private final AthenzService athenzService; + private final Clock clock; + + AthenzCredentialsService(IdentityConfig identityConfig, + IdentityDocumentService identityDocumentService, + AthenzService athenzService, + Clock clock) { + this.identityConfig = identityConfig; + this.identityDocumentService = identityDocumentService; + this.athenzService = athenzService; + this.clock = clock; + } + + AthenzCredentials registerInstance() { + KeyPair keyPair = CryptoUtils.createKeyPair(); + String rawDocument = identityDocumentService.getSignedIdentityDocument(); + SignedIdentityDocument document = parseSignedIdentityDocument(rawDocument); + PKCS10CertificationRequest csr = CryptoUtils.createCSR(identityConfig.domain(), + identityConfig.service(), + document.dnsSuffix, + document.providerUniqueId, + keyPair); + InstanceRegisterInformation instanceRegisterInformation = + new InstanceRegisterInformation(document.providerService, + identityConfig.domain(), + identityConfig.service(), + rawDocument, + CryptoUtils.toPem(csr)); + InstanceIdentity instanceIdentity = athenzService.sendInstanceRegisterRequest(instanceRegisterInformation, + document.ztsEndpoint); + return toAthenzCredentials(instanceIdentity, keyPair, document); + } + + AthenzCredentials updateCredentials(AthenzCredentials currentCredentials) { + SignedIdentityDocument document = currentCredentials.getIdentityDocument(); + KeyPair newKeyPair = CryptoUtils.createKeyPair(); + PKCS10CertificationRequest csr = CryptoUtils.createCSR(identityConfig.domain(), + identityConfig.service(), + document.dnsSuffix, + document.providerUniqueId, + newKeyPair); + InstanceRefreshInformation refreshInfo = new InstanceRefreshInformation(CryptoUtils.toPem(csr)); + InstanceIdentity instanceIdentity = + athenzService.sendInstanceRefreshRequest(document.providerService, + identityConfig.domain(), + identityConfig.service(), + document.providerUniqueId, + refreshInfo, + document.ztsEndpoint, + currentCredentials.getCertificate(), + currentCredentials.getKeyPair().getPrivate()); + return toAthenzCredentials(instanceIdentity, newKeyPair, document); + } + + private AthenzCredentials toAthenzCredentials(InstanceIdentity instanceIdentity, + KeyPair keyPair, + SignedIdentityDocument identityDocument) { + X509Certificate certificate = instanceIdentity.getX509Certificate(); + String serviceToken = instanceIdentity.getServiceToken(); + return new AthenzCredentials(serviceToken, certificate, keyPair, identityDocument, clock.instant()); + } + + private static SignedIdentityDocument parseSignedIdentityDocument(String rawDocument) { + try { + return mapper.readValue(rawDocument, SignedIdentityDocument.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java index d2c914fc209..356780a0900 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImpl.java @@ -1,75 +1,237 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.athenz.impl; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; +import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; +import com.yahoo.log.LogLevel; -import java.io.IOException; -import java.security.KeyPair; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; /** * @author mortent + * @author bjorncs */ public final class AthenzIdentityProviderImpl extends AbstractComponent implements AthenzIdentityProvider { - private final ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger log = Logger.getLogger(AthenzIdentityProviderImpl.class.getName()); - private InstanceIdentity instanceIdentity; + // TODO Make some of these values configurable through config. Match requested expiration of register/update requests. + // TODO These should match the requested expiration + static final Duration EXPIRES_AFTER = Duration.ofDays(1); + static final Duration EXPIRATION_MARGIN = Duration.ofMinutes(30); + static final Duration INITIAL_WAIT_NTOKEN = Duration.ofMinutes(5); + static final Duration UPDATE_PERIOD = EXPIRES_AFTER.dividedBy(3); + static final Duration REDUCED_UPDATE_PERIOD = Duration.ofMinutes(30); + static final Duration INITIAL_BACKOFF_DELAY = Duration.ofMinutes(4); + static final Duration MAX_REGISTER_BACKOFF_DELAY = Duration.ofHours(1); + static final int BACKOFF_DELAY_MULTIPLIER = 2; + static final Duration AWAIT_TERMINTATION_TIMEOUT = Duration.ofSeconds(90); - private final String dnsSuffix; - private final String providerUniqueId; + + static final String REGISTER_INSTANCE_TAG = "register-instance"; + static final String UPDATE_CREDENTIALS_TAG = "update-credentials"; + static final String TIMEOUT_INITIAL_WAIT_TAG = "timeout-initial-wait"; + + + private final AtomicReference<AthenzCredentials> credentials = new AtomicReference<>(); + private final AtomicReference<Throwable> lastThrowable = new AtomicReference<>(); + private final CountDownLatch credentialsRetrievedSignal = new CountDownLatch(1); + private final AthenzCredentialsService athenzCredentialsService; + private final Scheduler scheduler; + private final Clock clock; private final String domain; private final String service; @Inject - public AthenzIdentityProviderImpl(IdentityConfig config) throws IOException { - this(config, new ServiceProviderApi(config.loadBalancerAddress()), new AthenzService()); + public AthenzIdentityProviderImpl(IdentityConfig config) { + this(config, + new AthenzCredentialsService(config, + new IdentityDocumentService(config.loadBalancerAddress()), + new AthenzService(), + Clock.systemUTC()), + new ThreadPoolScheduler(), + Clock.systemUTC()); } // Test only AthenzIdentityProviderImpl(IdentityConfig config, - ServiceProviderApi serviceProviderApi, - AthenzService athenzService) throws IOException { - KeyPair keyPair = CryptoUtils.createKeyPair(); + AthenzCredentialsService athenzCredentialsService, + Scheduler scheduler, + Clock clock) { + this.athenzCredentialsService = athenzCredentialsService; + this.scheduler = scheduler; + this.clock = clock; this.domain = config.domain(); this.service = config.service(); - String rawDocument = serviceProviderApi.getSignedIdentityDocument(); - SignedIdentityDocument document = objectMapper.readValue(rawDocument, SignedIdentityDocument.class); - this.dnsSuffix = document.dnsSuffix; - this.providerUniqueId = document.providerUniqueId; - - InstanceRegisterInformation instanceRegisterInformation = new InstanceRegisterInformation( - document.providerService, - this.domain, - this.service, - rawDocument, - CryptoUtils.toPem(CryptoUtils.createCSR(domain, service, dnsSuffix, providerUniqueId, keyPair)), - true - ); - instanceIdentity = athenzService.sendInstanceRegisterRequest( instanceRegisterInformation, document.ztsEndpoint); + scheduler.submit(new RegisterInstanceTask()); + scheduler.schedule(new TimeoutInitialWaitTask(), INITIAL_WAIT_NTOKEN); } @Override public String getNToken() { - return instanceIdentity.getServiceToken(); + try { + credentialsRetrievedSignal.await(); + AthenzCredentials credentialsSnapshot = credentials.get(); + if (credentialsSnapshot == null) { + throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", lastThrowable.get()); + } + if (isExpired(credentialsSnapshot)) { + throw new AthenzIdentityProviderException("Athenz credentials are expired", lastThrowable.get()); + } + return credentialsSnapshot.getNToken(); + } catch (InterruptedException e) { + throw new AthenzIdentityProviderException("Failed to register instance credentials", lastThrowable.get()); + } } @Override - public String getX509Cert() { - return instanceIdentity.getX509Certificate(); + public String getDomain() { + return domain; } @Override - public String domain() { - return domain; + public String getService() { + return service; } @Override - public String service() { - return service; + public void deconstruct() { + scheduler.shutdown(AWAIT_TERMINTATION_TIMEOUT); + } + + private boolean isExpired(AthenzCredentials credentials) { + return clock.instant().isAfter(getExpirationTime(credentials)); + } + + private static Instant getExpirationTime(AthenzCredentials credentials) { + return credentials.getCreatedAt().plus(EXPIRES_AFTER).minus(EXPIRATION_MARGIN); + } + + private class RegisterInstanceTask implements RunnableWithTag { + + private final Duration backoffDelay; + + RegisterInstanceTask() { + this(INITIAL_BACKOFF_DELAY); + } + + RegisterInstanceTask(Duration backoffDelay) { + this.backoffDelay = backoffDelay; + } + + @Override + public void run() { + try { + credentials.set(athenzCredentialsService.registerInstance()); + credentialsRetrievedSignal.countDown(); + scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD); + } catch (Throwable t) { + log.log(LogLevel.ERROR, "Failed to register instance: " + t.getMessage(), t); + lastThrowable.set(t); + Duration nextBackoffDelay = backoffDelay.multipliedBy(BACKOFF_DELAY_MULTIPLIER); + if (nextBackoffDelay.compareTo(MAX_REGISTER_BACKOFF_DELAY) > 0) { + nextBackoffDelay = MAX_REGISTER_BACKOFF_DELAY; + } + scheduler.schedule(new RegisterInstanceTask(nextBackoffDelay), backoffDelay); + } + } + + @Override + public String tag() { + return REGISTER_INSTANCE_TAG; + } + } + + private class UpdateCredentialsTask implements RunnableWithTag { + @Override + public void run() { + AthenzCredentials currentCredentials = credentials.get(); + try { + AthenzCredentials newCredentials = isExpired(currentCredentials) + ? athenzCredentialsService.registerInstance() + : athenzCredentialsService.updateCredentials(currentCredentials); + credentials.set(newCredentials); + scheduler.schedule(new UpdateCredentialsTask(), UPDATE_PERIOD); + } catch (Throwable t) { + log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); + lastThrowable.set(t); + Duration timeToExpiration = Duration.between(clock.instant(), getExpirationTime(currentCredentials)); + // NOTE: Update period might be after timeToExpiration, still we do not want to DDoS Athenz. + Duration updatePeriod = + timeToExpiration.compareTo(UPDATE_PERIOD) > 0 ? UPDATE_PERIOD : REDUCED_UPDATE_PERIOD; + scheduler.schedule(new UpdateCredentialsTask(), updatePeriod); + } + } + + @Override + public String tag() { + return UPDATE_CREDENTIALS_TAG; + } + } + + private class TimeoutInitialWaitTask implements RunnableWithTag { + @Override + public void run() { + credentialsRetrievedSignal.countDown(); + } + + @Override + public String tag() { + return TIMEOUT_INITIAL_WAIT_TAG; + } } + + private static class ThreadPoolScheduler implements Scheduler { + + private static final Logger log = Logger.getLogger(ThreadPoolScheduler.class.getName()); + + private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(0); + + @Override + public void schedule(RunnableWithTag runnable, Duration delay) { + log.log(LogLevel.FINE, String.format("Scheduling task '%s' in '%s'", runnable.tag(), delay)); + executor.schedule(runnable, delay.getSeconds(), TimeUnit.SECONDS); + } + + @Override + public void submit(RunnableWithTag runnable) { + log.log(LogLevel.FINE, String.format("Scheduling task '%s' now", runnable.tag())); + executor.submit(runnable); + } + + @Override + public void shutdown(Duration timeout) { + try { + executor.shutdownNow(); + executor.awaitTermination(AWAIT_TERMINTATION_TIMEOUT.getSeconds(), TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + } + + public interface Scheduler { + void schedule(RunnableWithTag runnable, Duration delay); + default void submit(RunnableWithTag runnable) { schedule(runnable, Duration.ZERO); } + default void shutdown(Duration timeout) {} + } + + public interface RunnableWithTag extends Runnable { + + String tag(); + } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzService.java index dc1f8956def..898f90e3438 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzService.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/AthenzService.java @@ -3,6 +3,7 @@ package com.yahoo.container.jdisc.athenz.impl; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -10,6 +11,7 @@ import org.apache.http.conn.ssl.SSLContextBuilder; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; import org.eclipse.jetty.http.HttpStatus; @@ -17,6 +19,7 @@ import org.eclipse.jetty.http.HttpStatus; import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; import java.security.KeyManagementException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -33,18 +36,19 @@ import java.security.cert.X509Certificate; */ public class AthenzService { - private static final String INSTANCE_API_PATH = "zts/v1/instance"; + private static final String INSTANCE_API_PATH = "/zts/v1/instance"; private final ObjectMapper objectMapper = new ObjectMapper(); + private final HttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true); /** * Send instance register request to ZTS, get InstanceIdentity */ public InstanceIdentity sendInstanceRegisterRequest(InstanceRegisterInformation instanceRegisterInformation, - String ztsEndpoint) { - try(CloseableHttpClient client = HttpClientBuilder.create().build()) { + URI uri) { + try(CloseableHttpClient client = HttpClientBuilder.create().setRetryHandler(retryHandler).build()) { HttpUriRequest postRequest = RequestBuilder.post() - .setUri(ztsEndpoint + INSTANCE_API_PATH) + .setUri(uri.resolve(INSTANCE_API_PATH)) .setEntity(toJsonStringEntity(instanceRegisterInformation)) .build(); return getInstanceIdentity(client, postRequest); @@ -58,13 +62,16 @@ public class AthenzService { String instanceServiceName, String instanceId, InstanceRefreshInformation instanceRefreshInformation, - String ztsEndpoint, + URI ztsEndpoint, X509Certificate certicate, PrivateKey privateKey) { - try (CloseableHttpClient client = createHttpClientWithTlsAuth(certicate, privateKey)) { - String uri = String.format("%s/%s/%s/%s/%s", - ztsEndpoint + INSTANCE_API_PATH, - providerService, instanceDomain, instanceServiceName, instanceId); + try (CloseableHttpClient client = createHttpClientWithTlsAuth(certicate, privateKey, retryHandler)) { + URI uri = ztsEndpoint + .resolve(INSTANCE_API_PATH + '/') + .resolve(providerService + '/') + .resolve(instanceDomain + '/') + .resolve(instanceServiceName + '/') + .resolve(instanceId); HttpUriRequest postRequest = RequestBuilder.post() .setUri(uri) .setEntity(toJsonStringEntity(instanceRefreshInformation)) @@ -92,7 +99,9 @@ public class AthenzService { return new StringEntity(objectMapper.writeValueAsString(value), ContentType.APPLICATION_JSON); } - private static CloseableHttpClient createHttpClientWithTlsAuth(X509Certificate certificate, PrivateKey privateKey) { + private static CloseableHttpClient createHttpClientWithTlsAuth(X509Certificate certificate, + PrivateKey privateKey, + HttpRequestRetryHandler retryHandler) { try { String dummyPassword = "athenz"; KeyStore keyStore = KeyStore.getInstance("JKS"); @@ -102,6 +111,7 @@ public class AthenzService { .loadKeyMaterial(keyStore, dummyPassword.toCharArray()) .build(); return HttpClientBuilder.create() + .setRetryHandler(retryHandler) .setSslcontext(sslContext) .build(); } catch (KeyStoreException | UnrecoverableKeyException | NoSuchAlgorithmException | diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtils.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtils.java index 1b109e4bacb..388b40a1fe0 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtils.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/CryptoUtils.java @@ -6,6 +6,9 @@ import org.bouncycastle.asn1.x509.Extension; import org.bouncycastle.asn1.x509.ExtensionsGenerator; import org.bouncycastle.asn1.x509.GeneralName; import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openssl.PEMParser; import org.bouncycastle.openssl.jcajce.JcaPEMWriter; import org.bouncycastle.operator.OperatorCreationException; @@ -23,6 +26,7 @@ import java.io.UncheckedIOException; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; import java.security.cert.X509Certificate; /** @@ -30,6 +34,8 @@ import java.security.cert.X509Certificate; */ class CryptoUtils { + private static final BouncyCastleProvider bouncyCastleProvider = new BouncyCastleProvider(); + private CryptoUtils() {} static KeyPair createKeyPair() { @@ -45,7 +51,7 @@ class CryptoUtils { String identityService, String dnsSuffix, String providerUniqueId, - KeyPair keyPair) throws IOException { + KeyPair keyPair) { try { // Add SAN dnsname <service>.<domain-with-dashes>.<provider-dnsname-suffix> // and SAN dnsname <provider-unique-instance-id>.instanceid.athenz.<provider-dnsname-suffix> @@ -71,6 +77,8 @@ class CryptoUtils { return requestBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate())); } catch (OperatorCreationException e) { throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); } } @@ -87,12 +95,19 @@ class CryptoUtils { static X509Certificate parseCertificate(String pemEncodedCertificate) { try (PEMParser parser = new PEMParser(new StringReader(pemEncodedCertificate))) { Object pemObject = parser.readObject(); - if (!(pemObject instanceof X509Certificate)) { - throw new IllegalArgumentException("Expeceted X509Certificate instance, got " + pemObject); + if (pemObject instanceof X509Certificate) { + return (X509Certificate) pemObject; } - return (X509Certificate) pemObject; + if (pemObject instanceof X509CertificateHolder) { + return new JcaX509CertificateConverter() + .setProvider(bouncyCastleProvider) + .getCertificate((X509CertificateHolder) pemObject); + } + throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject); } catch (IOException e) { throw new UncheckedIOException(e); + } catch (CertificateException e) { + throw new RuntimeException(e); } } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/ServiceProviderApi.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/IdentityDocumentService.java index 6c1c22d07e0..542a5c739c8 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/ServiceProviderApi.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/IdentityDocumentService.java @@ -3,8 +3,8 @@ package com.yahoo.container.jdisc.athenz.impl; import com.yahoo.vespa.defaults.Defaults; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContextBuilder; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; @@ -15,20 +15,21 @@ import org.eclipse.jetty.http.HttpStatus; import java.io.IOException; import java.net.URI; -import java.net.URLEncoder; +import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; /** * @author mortent + * @author bjorncs */ -public class ServiceProviderApi { +public class IdentityDocumentService { - private final URI providerUri; + private final URI identityDocumentApiUri; - public ServiceProviderApi(String providerAddress) { - providerUri = URI.create(String.format("https://%s:8443/athenz/v1/provider", providerAddress)); + public IdentityDocumentService(String loadBalancerName) { + this.identityDocumentApiUri = createIdentityDocumentApiUri(loadBalancerName); } /** @@ -36,11 +37,7 @@ public class ServiceProviderApi { */ public String getSignedIdentityDocument() { try (CloseableHttpClient httpClient = createHttpClient()) { - // TODO Figure out a proper way of determining the hostname matching what's registred in node-repository - String uri = providerUri + "/identity-document?hostname=" + URLEncoder.encode( - Defaults.getDefaults().vespaHostname(), "UTF-8"); - HttpUriRequest request = RequestBuilder.get().setUri(uri).build(); - CloseableHttpResponse idDocResponse = httpClient.execute(request); + CloseableHttpResponse idDocResponse = httpClient.execute(new HttpGet(identityDocumentApiUri)); String responseContent = EntityUtils.toString(idDocResponse.getEntity()); if (HttpStatus.isSuccess(idDocResponse.getStatusLine().getStatusCode())) { return responseContent; @@ -70,4 +67,19 @@ public class ServiceProviderApi { } } + private static URI createIdentityDocumentApiUri(String loadBalancerName) { + try { + // TODO Figure out a proper way of determining the hostname matching what's registred in node-repository + return new URIBuilder() + .setScheme("https") + .setHost(loadBalancerName) + .setPort(4443) + .setPath("/athenz/v1/provider/identity-document") + .addParameter("hostname", Defaults.getDefaults().vespaHostname()) + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceIdentity.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceIdentity.java index ccb9b12c61a..20bbb2aa67e 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceIdentity.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceIdentity.java @@ -4,8 +4,13 @@ package com.yahoo.container.jdisc.athenz.impl; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.util.Map; +import java.io.IOException; +import java.security.cert.X509Certificate; /** * Used for deserializing response from ZTS @@ -15,42 +20,29 @@ import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class InstanceIdentity { - @JsonProperty("attributes") private final Map<String, String> attributes; - @JsonProperty("provider") private final String provider; - @JsonProperty("name") private final String name; - @JsonProperty("instanceId") private final String instanceId; - @JsonProperty("x509Certificate") private final String x509Certificate; - @JsonProperty("x509CertificateSigner") private final String x509CertificateSigner; - @JsonProperty("sshCertificate") private final String sshCertificate; - @JsonProperty("sshCertificateSigner") private final String sshCertificateSigner; + @JsonProperty("x509Certificate") private final X509Certificate x509Certificate; @JsonProperty("serviceToken") private final String serviceToken; - public InstanceIdentity( - @JsonProperty("attributes") Map<String, String> attributes, - @JsonProperty("provider") String provider, - @JsonProperty("name") String name, - @JsonProperty("instanceId") String instanceId, - @JsonProperty("x509Certificate") String x509Certificate, - @JsonProperty("x509CertificateSigner") String x509CertificateSigner, - @JsonProperty("sshCertificate") String sshCertificate, - @JsonProperty("sshCertificateSigner") String sshCertificateSigner, - @JsonProperty("serviceToken") String serviceToken) { - this.attributes = attributes; - this.provider = provider; - this.name = name; - this.instanceId = instanceId; + public InstanceIdentity(@JsonProperty("x509Certificate") @JsonDeserialize(using = X509CertificateDeserializer.class) + X509Certificate x509Certificate, + @JsonProperty("serviceToken") String serviceToken) { this.x509Certificate = x509Certificate; - this.x509CertificateSigner = x509CertificateSigner; - this.sshCertificate = sshCertificate; - this.sshCertificateSigner = sshCertificateSigner; this.serviceToken = serviceToken; } - public String getX509Certificate() { + public X509Certificate getX509Certificate() { return x509Certificate; } public String getServiceToken() { return serviceToken; } + + public static class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> { + @Override + public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException { + return CryptoUtils.parseCertificate(parser.getValueAsString()); + } + } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRefreshInformation.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRefreshInformation.java index 621eafca3bb..dd893cb3143 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRefreshInformation.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRefreshInformation.java @@ -15,10 +15,9 @@ public class InstanceRefreshInformation { @JsonProperty("csr") private final String csr; @JsonProperty("token") - private final boolean requestServiceToken; + private final boolean requestServiceToken = true; - public InstanceRefreshInformation(String csr, boolean requestServiceToken) { + public InstanceRefreshInformation(String csr) { this.csr = csr; - this.requestServiceToken = requestServiceToken; } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRegisterInformation.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRegisterInformation.java index 61ab810abd5..e2355cb7a2d 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRegisterInformation.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/InstanceRegisterInformation.java @@ -26,14 +26,13 @@ public class InstanceRegisterInformation { @JsonProperty("csr") private final String csr; @JsonProperty("token") - private final boolean token; + private final boolean token = true; - public InstanceRegisterInformation(String provider, String domain, String service, String attestationData, String csr, boolean token) { + public InstanceRegisterInformation(String provider, String domain, String service, String attestationData, String csr) { this.provider = provider; this.domain = domain; this.service = service; this.attestationData = attestationData; this.csr = csr; - this.token = token; } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/SignedIdentityDocument.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/SignedIdentityDocument.java index d302b3d96ce..5d5b5430859 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/SignedIdentityDocument.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/impl/SignedIdentityDocument.java @@ -5,21 +5,24 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.net.URI; + /** * @author bjorncs */ +// TODO Most of these value should ideally be config provided by config-model @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) class SignedIdentityDocument { public final String providerUniqueId; public final String dnsSuffix; public final String providerService; - public final String ztsEndpoint; + public final URI ztsEndpoint; public SignedIdentityDocument(@JsonProperty("provider-unique-id") String providerUniqueId, @JsonProperty("dns-suffix") String dnsSuffix, @JsonProperty("provider-service") String providerService, - @JsonProperty("zts-endpoint") String ztsEndpoint) { + @JsonProperty("zts-endpoint") URI ztsEndpoint) { this.providerUniqueId = providerUniqueId; this.dnsSuffix = dnsSuffix; this.providerService = providerService; diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java index 1f64fb0d379..1c0efef2089 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/athenz/impl/AthenzIdentityProviderImplTest.java @@ -3,11 +3,30 @@ package com.yahoo.container.jdisc.athenz.impl; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; -import org.junit.Assert; +import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.RunnableWithTag; +import com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.Scheduler; +import com.yahoo.test.ManualClock; import org.junit.Test; -import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.PriorityQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.INITIAL_BACKOFF_DELAY; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.INITIAL_WAIT_NTOKEN; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.MAX_REGISTER_BACKOFF_DELAY; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.REDUCED_UPDATE_PERIOD; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.REGISTER_INSTANCE_TAG; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.TIMEOUT_INITIAL_WAIT_TAG; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.UPDATE_CREDENTIALS_TAG; +import static com.yahoo.container.jdisc.athenz.impl.AthenzIdentityProviderImpl.UPDATE_PERIOD; +import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; @@ -15,25 +34,112 @@ import static org.mockito.Mockito.when; /** * @author mortent + * @author bjorncs */ public class AthenzIdentityProviderImplTest { + private static final IdentityConfig IDENTITY_CONFIG = + new IdentityConfig(new IdentityConfig.Builder() + .service("tenantService").domain("tenantDomain").loadBalancerAddress("cfg")); + + @Test + public void athenz_credentials_are_retrieved_after_component_contruction_completed() { + IdentityDocumentService identityDocumentService = mock(IdentityDocumentService.class); + AthenzService athenzService = mock(AthenzService.class); + ManualClock clock = new ManualClock(Instant.EPOCH); + MockScheduler scheduler = new MockScheduler(clock); + + when(identityDocumentService.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); + when(athenzService.sendInstanceRegisterRequest(any(), any())).thenReturn( + new InstanceIdentity(null, "TOKEN")); + AthenzCredentialsService credentialService = + new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, clock); + + AthenzIdentityProvider identityProvider = + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, credentialService, scheduler, clock); + + List<MockScheduler.CompletedTask> expectedTasks = + Arrays.asList( + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, Duration.ZERO), + new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN)); + // Don't run update credential tasks, otherwise infinite loop + List<MockScheduler.CompletedTask> completedTasks = + scheduler.runAllTasks(task -> !task.tag().equals(UPDATE_CREDENTIALS_TAG)); + assertEquals(expectedTasks, completedTasks); + assertEquals("TOKEN", identityProvider.getNToken()); + } + @Test - public void ntoken_fetched_on_init() throws IOException { - IdentityConfig config = new IdentityConfig(new IdentityConfig.Builder().service("tenantService").domain("tenantDomain").loadBalancerAddress("cfg")); - ServiceProviderApi serviceProviderApi = mock(ServiceProviderApi.class); + public void register_instance_uses_exponential_backoff() { + AthenzCredentialsService credentialService = mock(AthenzCredentialsService.class); + when(credentialService.registerInstance()) + .thenThrow(new RuntimeException("#1")) + .thenThrow(new RuntimeException("#2")) + .thenThrow(new RuntimeException("#3")) + .thenThrow(new RuntimeException("#4")) + .thenThrow(new RuntimeException("#5")) + .thenReturn(new AthenzCredentials("TOKEN", null, null, null, Instant.now())); + + ManualClock clock = new ManualClock(Instant.EPOCH); + MockScheduler scheduler = new MockScheduler(clock); + AthenzIdentityProvider identityProvider = + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, credentialService, scheduler, clock); + + List<MockScheduler.CompletedTask> expectedTasks = + Arrays.asList( + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, Duration.ZERO), + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY), + new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN), + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY.multipliedBy(2)), + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY.multipliedBy(4)), + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, INITIAL_BACKOFF_DELAY.multipliedBy(8)), + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, MAX_REGISTER_BACKOFF_DELAY)); + // Don't run update credential tasks, otherwise infinite loop + List<MockScheduler.CompletedTask> completedTasks = + scheduler.runAllTasks(task -> !task.tag().equals(UPDATE_CREDENTIALS_TAG)); + assertEquals(expectedTasks, completedTasks); + assertEquals("TOKEN", identityProvider.getNToken()); + } + + @Test + public void failed_credentials_updates_will_schedule_retries() { + IdentityDocumentService identityDocumentService = mock(IdentityDocumentService.class); AthenzService athenzService = mock(AthenzService.class); + ManualClock clock = new ManualClock(Instant.EPOCH); + MockScheduler scheduler = new MockScheduler(clock); - when(serviceProviderApi.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); - when(athenzService.sendInstanceRegisterRequest(any(), anyString())).thenReturn( - new InstanceIdentity(null, null, null, null, null, null, null, null, "TOKEN")); + when(identityDocumentService.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); + when(athenzService.sendInstanceRegisterRequest(any(), any())).thenReturn( + new InstanceIdentity(null, "TOKEN")); + when(athenzService.sendInstanceRefreshRequest(anyString(), anyString(), anyString(), + anyString(), any(), any(), any(), any())) + .thenThrow(new RuntimeException("#1")) + .thenThrow(new RuntimeException("#2")) + .thenThrow(new RuntimeException("#3")) + .thenReturn(new InstanceIdentity(null, "TOKEN")); + AthenzCredentialsService credentialService = + new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, clock); - AthenzIdentityProvider identityProvider = new AthenzIdentityProviderImpl(config, serviceProviderApi, athenzService); + AthenzIdentityProvider identityProvider = + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, credentialService, scheduler, clock); - Assert.assertEquals("TOKEN", identityProvider.getNToken()); + List<MockScheduler.CompletedTask> expectedTasks = + Arrays.asList( + new MockScheduler.CompletedTask(REGISTER_INSTANCE_TAG, Duration.ZERO), + new MockScheduler.CompletedTask(TIMEOUT_INITIAL_WAIT_TAG, INITIAL_WAIT_NTOKEN), + new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD), + new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD), + new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, REDUCED_UPDATE_PERIOD), + new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, REDUCED_UPDATE_PERIOD), + new MockScheduler.CompletedTask(UPDATE_CREDENTIALS_TAG, UPDATE_PERIOD)); + AtomicInteger counter = new AtomicInteger(0); + List<MockScheduler.CompletedTask> completedTasks = + scheduler.runAllTasks(task -> counter.getAndIncrement() < 7); // 1 registration + 1 timeout + 5 update tasks + assertEquals(expectedTasks, completedTasks); + assertEquals("TOKEN", identityProvider.getNToken()); } - private String getIdentityDocument() { + private static String getIdentityDocument() { return "{\n" + " \"identity-document\": \"eyJwcm92aWRlci11bmlxdWUtaWQiOnsidGVuYW50IjoidGVuYW50IiwiYXBwbGljYXRpb24iOiJhcHBsaWNhdGlvbiIsImVudmlyb25tZW50IjoiZGV2IiwicmVnaW9uIjoidXMtbm9ydGgtMSIsImluc3RhbmNlIjoiZGVmYXVsdCIsImNsdXN0ZXItaWQiOiJkZWZhdWx0IiwiY2x1c3Rlci1pbmRleCI6MH0sImNvbmZpZ3NlcnZlci1ob3N0bmFtZSI6ImxvY2FsaG9zdCIsImluc3RhbmNlLWhvc3RuYW1lIjoieC55LmNvbSIsImNyZWF0ZWQtYXQiOjE1MDg3NDgyODUuNzQyMDAwMDAwfQ==\",\n" + " \"signature\": \"kkEJB/98cy1FeXxzSjtvGH2a6BFgZu/9/kzCcAqRMZjENxnw5jyO1/bjZVzw2Sz4YHPsWSx2uxb32hiQ0U8rMP0zfA9nERIalSP0jB/hMU8laezGhdpk6VKZPJRC6YKAB9Bsv2qUIfMsSxkMqf66GUvjZAGaYsnNa2yHc1jIYHOGMeJO+HNPYJjGv26xPfAOPIKQzs3RmKrc3FoweTCsIwm5oblqekdJvVWYe0obwlOSB5uwc1zpq3Ie1QBFtJRuCGMVHg1pDPxXKBHLClGIrEvzLmICy6IRdHszSO5qiwujUD7sbrbM0sB/u0cYucxbcsGRUmBvme3UAw2mW9POVQ==\",\n" + @@ -46,4 +152,82 @@ public class AthenzIdentityProviderImplTest { "}"; } + + private static class MockScheduler implements Scheduler { + + private final PriorityQueue<DelayedTask> tasks = new PriorityQueue<>(); + private final ManualClock clock; + + MockScheduler(ManualClock clock) { + this.clock = clock; + } + + @Override + public void schedule(RunnableWithTag task, Duration delay) { + tasks.offer(new DelayedTask(task, delay, clock.instant().plus(delay))); + } + + List<CompletedTask> runAllTasks(Predicate<RunnableWithTag> filter) { + List<CompletedTask> completedTasks = new ArrayList<>(); + while (!tasks.isEmpty()) { + DelayedTask task = tasks.poll(); + RunnableWithTag runnable = task.runnableWithTag; + if (filter.test(runnable)) { + clock.setInstant(task.startTime); + runnable.run(); + completedTasks.add(new CompletedTask(runnable.tag(), task.delay)); + } + } + return completedTasks; + } + + private static class DelayedTask implements Comparable<DelayedTask> { + final RunnableWithTag runnableWithTag; + final Duration delay; + final Instant startTime; + + DelayedTask(RunnableWithTag runnableWithTag, Duration delay, Instant startTime) { + this.runnableWithTag = runnableWithTag; + this.delay = delay; + this.startTime = startTime; + } + + @Override + public int compareTo(DelayedTask other) { + return this.startTime.compareTo(other.startTime); + } + } + + private static class CompletedTask { + final String tag; + final Duration delay; + + CompletedTask(String tag, Duration delay) { + this.tag = tag; + this.delay = delay; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CompletedTask that = (CompletedTask) o; + return Objects.equals(tag, that.tag) && + Objects.equals(delay, that.delay); + } + + @Override + public int hashCode() { + return Objects.hash(tag, delay); + } + + @Override + public String toString() { + return "CompletedTask{" + + "tag='" + tag + '\'' + + ", delay=" + delay + + '}'; + } + } + } } diff --git a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala b/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala index c83f6a63954..f0eff54dc16 100644 --- a/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala +++ b/container-jersey2/src/main/scala/com/yahoo/container/servlet/jersey/JerseyServletProvider.scala @@ -5,6 +5,7 @@ import java.io.{IOException, InputStream} import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jdk8.Jdk8Module +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider import com.yahoo.container.di.componentgraph.Provider import com.yahoo.container.di.config.RestApiContext @@ -21,7 +22,6 @@ import org.glassfish.jersey.servlet.ServletContainer import org.objectweb.asm.ClassReader import scala.collection.JavaConverters._ - import scala.util.control.Exception @@ -96,7 +96,10 @@ class JerseyServletProvider(restApiContext: RestApiContext) extends Provider[Ser def jacksonDatatypeJdk8Provider: JacksonJaxbJsonProvider = { val provider = new JacksonJaxbJsonProvider() - provider.setMapper(new ObjectMapper().registerModule(new Jdk8Module)) + provider.setMapper( + new ObjectMapper() + .registerModule(new Jdk8Module) + .registerModule(new JavaTimeModule)) provider } diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index eb05bdc672c..1513cf2213c 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -26,8 +26,12 @@ import java.util.List; */ public class QueryPacket extends Packet { - private Query query; + final private Query query; private QueryPacketData queryPacketData; + int sessionOffset = 0; // Start of sessionKey ignore section for cache key + int sessionSize = 0; // Length of sessionKey ignore section for cache key + int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key + int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key private QueryPacket(Query query) { this.query = query; @@ -73,6 +77,9 @@ public class QueryPacket extends Packet { return new byte[0]; } + private int getSessionKeySkipLength() { + return (sessionSize > 0) ? sessionSize + 4 : 0; + } /** * Returns an opaque cache key for the query represented by this * (pre-serialized) packet. @@ -86,35 +93,41 @@ public class QueryPacket extends Packet { // need to fiddle with feature flags to handle a non-existing // summary class. - int skipOffset = 4; // offset of offset/hits/timestamp fields - int skipLength = 12; // length of offset/hits/timestamp fields byte[] utf8Summary = getSummaryClassAsUtf8(); - byte[] stripped = new byte[encodedBody.length - skipLength + utf8Summary.length + 1]; - - System.arraycopy(encodedBody, 0, stripped, 0, skipOffset); - System.arraycopy(utf8Summary, 0, stripped, skipOffset, utf8Summary.length); - stripped[skipOffset + utf8Summary.length] = 0; - System.arraycopy(encodedBody, skipOffset + skipLength, - stripped, skipOffset + utf8Summary.length + 1, - encodedBody.length - (skipOffset + skipLength)); + byte[] stripped = new byte[encodedBody.length - (ignoreableSize + getSessionKeySkipLength()) + utf8Summary.length + 1]; + + System.arraycopy(encodedBody, 0, stripped, 0, ignoreableOffset); + stripped[1] = (byte)(stripped[1] & 0x7f); // Ignor sessionKey feature flag + System.arraycopy(utf8Summary, 0, stripped, ignoreableOffset, utf8Summary.length); + stripped[ignoreableOffset + utf8Summary.length] = 0; + + // Copy part up to sessionKey + System.arraycopy(encodedBody, ignoreableOffset + ignoreableSize, + stripped, ignoreableOffset + utf8Summary.length + 1, + sessionOffset - (ignoreableOffset + ignoreableSize)); + // Copy part after sessionKey + System.arraycopy(encodedBody, sessionOffset + getSessionKeySkipLength(), + stripped, utf8Summary.length + 1 + (sessionOffset - ignoreableSize), + encodedBody.length - (sessionOffset + getSessionKeySkipLength())); return stripped; } public void encodeBody(ByteBuffer buffer) { queryPacketData = new QueryPacketData(); - int startOfFieldToSave; + final int relativeZero = buffer.position(); boolean sendSessionKey = query.getGroupingSessionCache() || query.getRanking().getQueryCache(); int featureFlag = getFeatureInt(sendSessionKey); buffer.putInt(featureFlag); + ignoreableOffset = buffer.position() - relativeZero; IntegerCompressor.putCompressedPositiveNumber(getOffset(), buffer); IntegerCompressor.putCompressedPositiveNumber(getHits(), buffer); // store the cutoff time in the tag object, and then do a similar Math.max there buffer.putInt(Math.max(50, (int)query.getTimeLeft())); + ignoreableSize = buffer.position() - relativeZero - ignoreableOffset; buffer.putInt(getFlagInt()); - - startOfFieldToSave = buffer.position(); + int startOfFieldToSave = buffer.position(); Item.putString(query.getRanking().getProfile(), buffer); queryPacketData.setRankProfile(buffer, startOfFieldToSave); @@ -147,8 +160,10 @@ public class QueryPacket extends Packet { buffer.put(blob); } + sessionOffset = buffer.position() - relativeZero; if (sendSessionKey) { Utf8String key = query.getSessionId(true).asUtf8String(); + sessionSize = key.getByteLength(); buffer.putInt(key.getByteLength()); buffer.put(key.getBytes()); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java index 725ef89eadb..6af0d181695 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java @@ -14,8 +14,8 @@ import com.yahoo.fs4.QueryPacket; * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class CacheKey { - private int hashCode; - private byte[] serialized = null; + final private int hashCode; + final private byte[] serialized; /** * Create a cache key from the query packet. diff --git a/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java b/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java index 73a29745149..5134f33bc73 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java @@ -35,5 +35,5 @@ abstract class ParserBase extends Parser { } return name != null && arrayParameters.contains(name); } - + } diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index 988911fd408..1bd10234e55 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -127,9 +127,10 @@ final class ProgramParser { } - private yqlplusParser prepareParser(final String programName, CharStream input) { - yqlplusLexer lex = new yqlplusLexer(input); - lex.addErrorListener(new BaseErrorListener() { + private yqlplusParser prepareParser(String programName, CharStream input) { + yqlplusLexer lexer = new yqlplusLexer(input); + lexer.removeErrorListeners(); + lexer.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(@NotNull Recognizer<?, ?> recognizer, @@ -142,8 +143,10 @@ final class ProgramParser { } }); - TokenStream tokens = new CommonTokenStream(lex); + TokenStream tokens = new CommonTokenStream(lexer); + yqlplusParser parser = new yqlplusParser(tokens); + parser.removeErrorListeners(); parser.addErrorListener(new BaseErrorListener() { @Override @@ -224,6 +227,7 @@ final class ProgramParser { } static class Binding { + private final List<String> binding; Binding(String moduleName, String exportName) { @@ -245,9 +249,11 @@ final class ProgramParser { public List<String> toPathWith(List<String> rest) { return ImmutableList.copyOf(Iterables.concat(toPath(), rest)); } + } static class Scope { + final Scope root; final Scope parent; Set<String> cursors = ImmutableSet.of(); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 4ecae028f9d..259719571be 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -88,7 +88,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; * </p> * * @author Steinar Knutsen - * @author stiankri + * @author Stian Kristoffersen * @author Simon Thoresen */ @Beta diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java index 78bf14bae99..b771c64d1a9 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.fs4.GetDocSumsPacket; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.query.SessionId; import com.yahoo.search.result.Hit; import org.junit.Test; @@ -64,11 +65,13 @@ public class GetDocSumsPacketTestCase { @Test public void requireThatSessionIdIsEncodedAsPropertyWhenUsingSearchSession() throws BufferTooSmallException { Result result = new Result(new Query("?query=foo")); - result.getQuery().getSessionId(true); // create session id. + SessionId sessionId = result.getQuery().getSessionId(true); // create session id. result.getQuery().getRanking().setQueryCache(true); FastHit hit = new FastHit(); result.hits().add(hit); - assertPacket(false, result, new byte[] { 0, 0, 0, -115, 0, 0, 0, -37, 0, 0, 56, 17, 0, 0, 0, 0, + ByteBuffer answer = ByteBuffer.allocate(1024); + //assertEquals(0, sessionId.asUtf8String().getByteLength()); + answer.put(new byte[] { 0, 0, 0, (byte)(107+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 56, 17, 0, 0, 0, 0, // query timeout IGNORE, IGNORE, IGNORE, IGNORE, // "default" - rank profile @@ -78,13 +81,19 @@ public class GetDocSumsPacketTestCase { // 2 property entries 0, 0, 0, 2, // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 - 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd', 0, 0, 0, 34, 'q', 'r', 's', 'e', 'r', 'v', 'e', 'r', '.', - IGNORE, '.', IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, '.', IGNORE, '.','d', 'e', 'f', 'a', 'u', 'l', 't', + 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd'}); + answer.putInt(sessionId.asUtf8String().getByteLength()); + answer.put(sessionId.asUtf8String().getBytes()); + answer.put(new byte [] { // caches: features => true - 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e', + 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', + 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e', // flags - 0, 0, 0, 2 - }); + 0, 0, 0, 2}); + byte [] expected = new byte [answer.position()]; + answer.flip(); + answer.get(expected); + assertPacket(false, result, expected); } private static void assertPacket(boolean sendQuery, Hit hit, byte[] expected) throws BufferTooSmallException { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java index d87d32fa5cf..e7ab1cc137a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java @@ -26,4 +26,16 @@ public class CacheKeyTestCase extends junit.framework.TestCase { assertEquals(k1, k2); assertEquals(k1.hashCode(), k2.hashCode()); } + + public void testSessionKeyIgnored() { + Query a = new Query("/?query=abcd"); + QueryPacket ap = QueryPacket.create(a); + Query b = new Query("/?query=abcd&ranking.queryCache=true"); + QueryPacket bp = QueryPacket.create(b); + CacheKey ak = new CacheKey(ap); + CacheKey bk = new CacheKey(bp); + assertEquals(ak, bk); + assertEquals(ak.hashCode(), bk.hashCode()); + assertFalse(ap.getQueryPacketData().equals(bp.getQueryPacketData())); + } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 5a5e7586a32..75d965c6438 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -26,6 +26,7 @@ import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.request.AllOperation; import com.yahoo.search.grouping.request.EachOperation; import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.SessionId; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; @@ -295,8 +296,10 @@ public class FastSearcherTestCase { byte[] actual = new byte[buf.remaining()]; buf.get(actual); + SessionId sessionId = query.getSessionId(false); byte IGNORE = 69; - byte[] expected = new byte[] { 0, 0, 0, -77, 0, 0, 0, -37, 0, 0, 48, 17, 0, 0, 0, 0, + ByteBuffer answer = ByteBuffer.allocate(1024); + answer.put(new byte[] { 0, 0, 0, (byte)(145+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 48, 17, 0, 0, 0, 0, // query timeout IGNORE, IGNORE, IGNORE, IGNORE, // "default" - rank profile @@ -304,15 +307,20 @@ public class FastSearcherTestCase { // 3 property entries (rank, match, caches) 0, 0, 0, 3, // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 - 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd', 0, 0, 0, 34, 'q', 'r', 's', 'e', 'r', 'v', 'e', 'r', '.', - IGNORE, '.', IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, '.', IGNORE, '.','d', 'e', 'f', 'a', 'u', 'l', 't', + 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd'}); + answer.putInt(sessionId.asUtf8String().getBytes().length); + answer.put(sessionId.asUtf8String().getBytes()); + answer.put(new byte [] { // match: documentdb.searchdoctype => test 0, 0, 0, 5, 'm', 'a', 't', 'c', 'h', 0, 0, 0, 1, 0, 0, 0, 24, 'd', 'o', 'c', 'u', 'm', 'e', 'n', 't', 'd', 'b', '.', 's', 'e', 'a', 'r', 'c', 'h', 'd', 'o', 'c', 't', 'y', 'p', 'e', 0, 0, 0, 4, 't', 'e', 's', 't', // sessionId => qrserver.0.XXXXXXXXXXXXX.0 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e', // flags - 0, 0, 0, 2 - }; + 0, 0, 0, 2}); + byte [] expected = new byte [answer.position()]; + answer.flip(); + answer.get(expected); + assertEquals(expected.length, actual.length); for (int i = 0; i < expected.length; ++i) { if (expected[i] == IGNORE) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java index 257dfcc456d..b35ef0e09cf 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java @@ -50,7 +50,7 @@ public class PacketCacheTestCase extends junit.framework.TestCase { cache.setMaxCacheItemPercentage(50); - final int keysz = 30; + final int keysz = 36; // first control assumptions assertEquals(keysz, key1.byteSize()); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java index d8551898f7c..55215456379 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java @@ -14,6 +14,7 @@ import java.util.Optional; @JsonIgnoreProperties(ignoreUnknown = true) public class DeployOptions { + // TODO: Add build number here, so we can replace job timeout (12 hours) with triggering timeout (a few minutes?) public final Optional<ScrewdriverBuildJob> screwdriverBuildJob; public final Optional<String> vespaVersion; public final boolean ignoreValidationErrors; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java index df182b56fd8..37356f2c2a6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java @@ -19,8 +19,9 @@ public class Issue { private final String label; private final User assignee; private final PropertyId propertyId; + private final Type type; - private Issue(String summary, String description, String label, User assignee, PropertyId propertyId) { + private Issue(String summary, String description, String label, User assignee, PropertyId propertyId, Type type) { if (summary.isEmpty()) throw new IllegalArgumentException("Issue summary can not be empty!"); if (description.isEmpty()) throw new IllegalArgumentException("Issue description can not be empty!"); Objects.requireNonNull(propertyId, "An issue must belong to a property!"); @@ -30,26 +31,31 @@ public class Issue { this.label = label; this.assignee = assignee; this.propertyId = propertyId; + this.type = type; } public Issue(String summary, String description, PropertyId propertyId) { - this(summary, description, null, null, propertyId); + this(summary, description, null, null, propertyId, Type.defect); } public Issue append(String appendage) { - return new Issue(summary, description + appendage, label, assignee, propertyId); + return new Issue(summary, description + appendage, label, assignee, propertyId, type); } - public Issue withLabel(String label) { - return new Issue(summary, description, label, assignee, propertyId); + public Issue with(String label) { + return new Issue(summary, description, label, assignee, propertyId, type); } - public Issue withAssignee(User assignee) { - return new Issue(summary, description, label, assignee, propertyId); + public Issue with(User assignee) { + return new Issue(summary, description, label, assignee, propertyId, type); } - public Issue withPropertyId(PropertyId propertyId) { - return new Issue(summary, description, label, assignee, propertyId); + public Issue with(PropertyId propertyId) { + return new Issue(summary, description, label, assignee, propertyId, type); + } + + public Issue with(Type type) { + return new Issue(summary, description, label, assignee, propertyId, type); } public String summary() { @@ -72,4 +78,16 @@ public class Issue { return propertyId; } + public Type type() { + return type; + } + + + public enum Type { + + defect, // A defect which needs fixing. + task // A task the humans must perform. + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java new file mode 100644 index 00000000000..8ded0c5fb52 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/OwnershipIssues.java @@ -0,0 +1,48 @@ +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; + +import java.util.Optional; + +/** + * Periodically issues ownership confirmation requests for given applications, and escalates the issues if needed. + * + * Even machines wrought from cold steel occasionally require the gentle touch only a fleshling can provide. + * By making humans regularly acknowledge their dedication to given applications, this class provides the machine + * with reassurance that any misbehaving applications will swiftly be dealt with. + * Ignored confirmation requests are periodically redirected to humans of higher rank, until they are acknowledged. + * + * @author jvenstad + */ +public interface OwnershipIssues { + + /** + * Ensure ownership of the given application has been recently confirmed by the given property. + * + * @param issueId ID of the previous ownership issue filed for the given application. + * @param applicationId ID of the application for which to file an issue. + * @param propertyId ID of the property responsible for the given application. + * @return ID of the created issue, if one was created. + */ + Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, PropertyId propertyId); + + /** + * Ensure ownership of the given application has been recently confirmed by the given user. + * + * @param issueId ID of the previous ownership issue filed for the given application. + * @param applicationId ID of the application for which to file an issue. + * @param owner ID of the user responsible for the given application. + * @return ID of the created issue, if one was created. + */ + Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, User owner); + + /** + * Make sure the given ownership confirmation request is acted upon, unless it is already acknowledged. + * + * @param issueId ID of the ownership issue to escalate. + * @param propertyId ID of the property responsible for the issue, if any. + */ + void ensureResponse(IssueId issueId, Optional<PropertyId> propertyId); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummyOwnershipIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummyOwnershipIssues.java new file mode 100644 index 00000000000..0cf103739d1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummyOwnershipIssues.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; + +import java.util.Optional; + +public class DummyOwnershipIssues implements OwnershipIssues { + + @Override + public Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, PropertyId propertyId) { + return Optional.empty(); + } + + @Override + public Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, User owner) { + return Optional.empty(); + } + + @Override + public void ensureResponse(IssueId issueId, Optional<PropertyId> propertyId) { + + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java index 783a1267e64..b169194fd40 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java @@ -25,7 +25,8 @@ import java.util.logging.Logger; /** * A memory backed implementation of the Issues API which logs changes and does nothing else. * - * @author bratseth, jvenstad + * @author bratseth + * @author jvenstad */ public class LoggingDeploymentIssues implements DeploymentIssues { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index e7bdf786c8c..df6f023f6af 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -25,6 +25,7 @@ public interface ZoneRegistry { List<URI> getConfigServerUris(Environment environment, RegionName region); Optional<URI> getLogServerUri(Environment environment, RegionName region); Optional<Duration> getDeploymentTimeToLive(Environment environment, RegionName region); + Optional<RegionName> getDefaultRegion(Environment environment); URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application); URI getDashboardUri(); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 2e706f6df24..0cfcbc40601 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -152,6 +152,19 @@ <groupId>org.bouncycastle</groupId> <artifactId>bcprov-jdk15on</artifactId> </exclusion> + <!--Exclude all Jackson bundles provided by JDisc --> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> </exclusions> </dependency> @@ -164,6 +177,19 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </exclusion> + <!--Exclude all Jackson bundles provided by JDisc --> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> </exclusions> </dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 364e91f1828..e3400d76bce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -8,7 +8,11 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Change.VersionChange; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -38,26 +42,28 @@ public class Application { private final DeploymentJobs deploymentJobs; private final Optional<Change> deploying; private final boolean outstandingChange; + private final Optional<IssueId> ownershipIssueId; + private final ApplicationMetrics metrics; /** Creates an empty application */ public Application(ApplicationId id) { this(id, DeploymentSpec.empty, ValidationOverrides.empty, ImmutableMap.of(), new DeploymentJobs(Optional.empty(), Collections.emptyList(), Optional.empty()), - Optional.empty(), false); + Optional.empty(), false, Optional.empty(), new ApplicationMetrics(0, 0)); } /** Used from persistence layer: Do not use */ public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, - List<Deployment> deployments, - DeploymentJobs deploymentJobs, Optional<Change> deploying, boolean outstandingChange) { + List<Deployment> deployments, DeploymentJobs deploymentJobs, Optional<Change> deploying, + boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics) { this(id, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, d -> d)), - deploymentJobs, deploying, outstandingChange); + deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics); } Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map<Zone, Deployment> deployments, DeploymentJobs deploymentJobs, Optional<Change> deploying, - boolean outstandingChange) { + boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics) { Objects.requireNonNull(id, "id cannot be null"); Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null"); @@ -71,6 +77,8 @@ public class Application { this.deploymentJobs = deploymentJobs; this.deploying = deploying; this.outstandingChange = outstandingChange; + this.ownershipIssueId = ownershipIssueId; + this.metrics = metrics; } public ApplicationId id() { return id; } @@ -96,9 +104,9 @@ public class Application { * (deployments also includes manually deployed environments) */ public Map<Zone, Deployment> productionDeployments() { - return deployments.values().stream() - .filter(deployment -> deployment.zone().environment() == Environment.prod) - .collect(Collectors.toMap(Deployment::zone, Function.identity())); + return ImmutableMap.copyOf(deployments.values().stream() + .filter(deployment -> deployment.zone().environment() == Environment.prod) + .collect(Collectors.toMap(Deployment::zone, Function.identity()))); } public DeploymentJobs deploymentJobs() { return deploymentJobs; } @@ -115,39 +123,49 @@ public class Application { */ public boolean hasOutstandingChange() { return outstandingChange; } - /** + public Optional<IssueId> ownershipIssueId() { + return ownershipIssueId; + } + + public ApplicationMetrics metrics() { + return metrics; + } + + /** * Returns the oldest version this has deployed in a permanent zone (not test or staging), * or empty version if it is not deployed anywhere */ - public Optional<Version> deployedVersion() { + public Optional<Version> oldestDeployedVersion() { return productionDeployments().values().stream() - .sorted(Comparator.comparing(Deployment::version)) - .findFirst() - .map(Deployment::version); - } - - /** The version that should be used to compile this application */ - public Version compileVersion(Controller controller) { - return deployedVersion().orElse(controller.systemVersion()); + .map(Deployment::version) + .min(Comparator.naturalOrder()); } - /** Returns the version a deployment to this zone should use for this application */ - public Version currentDeployVersion(Controller controller, Zone zone) { - if ( ! deploying().isPresent()) - return currentVersion(controller, zone); - else if ( deploying().get() instanceof Change.ApplicationChange) - return currentVersion(controller, zone); - else + /** Returns the version a new deployment to this zone should use for this application */ + public Version deployVersionIn(Zone zone, Controller controller) { + if (deploying().isPresent() && deploying().get() instanceof VersionChange) return ((Change.VersionChange) deploying().get()).version(); + + return versionIn(zone, controller); } /** Returns the current version this application has, or if none; should use, in the given zone */ - public Version currentVersion(Controller controller, Zone zone) { - Deployment currentDeployment = deployments().get(zone); - if (currentDeployment != null) // Already deployed in this zone: Use that version - return currentDeployment.version(); + public Version versionIn(Zone zone, Controller controller) { + return Optional.ofNullable(deployments().get(zone)).map(Deployment::version) // Already deployed in this zone: Use that version + .orElse(oldestDeployedVersion().orElse(controller.systemVersion())); + } - return deployedVersion().orElse(controller.systemVersion()); + /** Returns the revision a new deployment to this zone should use for this application, or empty if we don't know */ + public Optional<ApplicationRevision> deployRevisionIn(Zone zone) { + if (deploying().isPresent() && deploying().get() instanceof Change.ApplicationChange) + return ((Change.ApplicationChange) deploying().get()).revision(); + + return revisionIn(zone); + } + + /** Returns the revision this application is or should be deployed with in the given zone, or empty if unknown. */ + public Optional<ApplicationRevision> revisionIn(Zone zone) { + return Optional.ofNullable(deployments().get(zone)).map(Deployment::revision); } @Override @@ -170,20 +188,8 @@ public class Application { return "application '" + id + "'"; } - /** Returns true if there is no current change to deploy - i.e deploying is empty or completely deployed */ - public boolean deployingCompleted() { - if ( ! deploying.isPresent()) return true; - return deploymentJobs().isDeployed(deploying.get()); - } - - /** Returns true if there is a current change which is blocked from being deployed to production at this instant */ - public boolean deployingBlocked(Instant instant) { - if ( ! deploying.isPresent()) return false; - return deploying.get().blockedBy(deploymentSpec, instant); - } - public boolean isBlocked(Instant instant) { - return ! deploymentSpec.canUpgradeAt(instant) || ! deploymentSpec.canChangeRevisionAt(instant); + return ! deploymentSpec().canUpgradeAt(instant) || ! deploymentSpec().canChangeRevisionAt(instant); } - + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 26debf3083f..841b9b4dd9f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; @@ -62,6 +63,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -109,14 +111,8 @@ public class ApplicationController { this.deploymentTrigger = new DeploymentTrigger(controller, curator, clock); - for (Application application : db.listApplications()) { - try (Lock lock = lock(application.id())) { - Optional<LockedApplication> lockedApplication = db.getApplication(application.id()) - .map(app -> new LockedApplication(app, lock)); - if ( ! lockedApplication.isPresent()) continue; // was removed since listing; ok - store(lockedApplication.get()); // re-write all applications to update storage format - } - } + for (Application application : db.listApplications()) + lockedIfPresent(application.id(), this::store); } /** Returns the application with the given id, or null if it is not present */ @@ -124,12 +120,6 @@ public class ApplicationController { return db.getApplication(id); } - - /** Returns an locked application with the given id that be updated and stored */ - public Optional<LockedApplication> get(ApplicationId id, Lock lock) { - return db.getApplication(id).map(application -> new LockedApplication(application, lock)); - } - /** * Returns the application with the given id * @@ -139,16 +129,6 @@ public class ApplicationController { return get(id).orElseThrow(() -> new IllegalArgumentException(id + " not found")); } - /** - * Returns a locked application that be updated and stored - * - * @throws IllegalArgumentException if it does not exist - * - */ - public LockedApplication require(ApplicationId id, Lock lock) { - return get(id, lock).orElseThrow(() -> new IllegalArgumentException(id + " not found")); - } - /** Returns a snapshot of all applications */ public List<Application> asList() { return db.listApplications(); @@ -251,6 +231,7 @@ public class ApplicationController { if ( ! (id.instance().value().equals("default") || id.instance().value().startsWith("default-pr"))) // TODO: Support instances properly throw new UnsupportedOperationException("Only the instance names 'default' and names starting with 'default-pr' are supported at the moment"); try (Lock lock = lock(id)) { + // TODO: Throwing is duplicated below. if (get(id).isPresent()) throw new IllegalArgumentException("An application with id '" + id + "' already exists"); @@ -288,22 +269,23 @@ public class ApplicationController { public ActivateResult deployApplication(ApplicationId applicationId, Zone zone, ApplicationPackage applicationPackage, DeployOptions options) { try (Lock lock = lock(applicationId)) { - // Determine what we are doing - LockedApplication application = get(applicationId, lock).orElse(new LockedApplication( + // TODO: Shouldn't this go through the above method? Seems you can cheat the checks here ... ? + LockedApplication application = get(applicationId).map(application1 -> new LockedApplication(application1, lock)).orElse(new LockedApplication( new Application(applicationId), lock) - ); + ); + // Determine what we are doing Version version; if (options.deployCurrentVersion) - version = application.currentVersion(controller, zone); + version = application.versionIn(zone, controller); else if (canDeployDirectlyTo(zone, options)) version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) return unexpectedDeployment(applicationId, zone, applicationPackage); else - version = application.currentDeployVersion(controller, zone); + version = application.deployVersionIn(zone, controller); - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); + Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.system(), zone); ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); if ( ! options.deployCurrentVersion) { @@ -314,17 +296,17 @@ public class ApplicationController { application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if ( ! canDeployDirectlyTo(zone, options) && jobType != null) { - // Update with (potentially) missing information about what we triggered - JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType); - application = application.with(application.deploymentJobs() - .withTriggering(jobType, - application.deploying(), - triggering.id(), - version, - Optional.of(revision), - triggering.reason(), - triggering.at())); + if ( ! canDeployDirectlyTo(zone, options) && jobType.isPresent()) { + // Update with (potentially) missing information about what we triggered: + // * When someone else triggered the job, we need to store a stand-in triggering event. + // * When this is the system test job, we need to record the new revision, for future use. + JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType.get()); + application = application.withJobTriggering(jobType.get(), + application.deploying(), + triggering.at(), + version, + Optional.of(revision), + triggering.reason()); } // Delete zones not listed in DeploymentSpec, if allowed @@ -357,14 +339,8 @@ public class ApplicationController { configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), applicationPackage.zippedContent()); preparedApplication.activate(); + application = application.withNewDeployment(zone, revision, version, clock.instant()); - // Use info from previous deployments is available - Deployment previousDeployment = application.deployments().getOrDefault(zone, new Deployment(zone, revision, version, clock.instant())); - Deployment newDeployment = new Deployment(zone, revision, version, clock.instant(), - previousDeployment.clusterUtils(), - previousDeployment.clusterInfo(), previousDeployment.metrics()); - - application = application.with(newDeployment); store(application); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse()); @@ -423,10 +399,9 @@ public class ApplicationController { * This is needed (only) in the case where some external entity triggers a job. */ private JobStatus.JobRun getOrCreateTriggering(Application application, Version version, DeploymentJobs.JobType jobType) { - if (jobType == null) return incompleteTriggeringEvent(version); JobStatus status = application.deploymentJobs().jobStatus().get(jobType); - if (status == null) return incompleteTriggeringEvent(version); - if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version); + if (status == null) return incompleteTriggeringEvent(version); + if ( ! status.lastTriggered().isPresent()) return incompleteTriggeringEvent(version); return status.lastTriggered().get(); } @@ -501,7 +476,7 @@ public class ApplicationController { return Optional.of(new InstanceEndpoints(endPointUrls)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e); + log.log(Level.FINE, "Failed to get endpoint information for " + deploymentId, e); return Optional.empty(); } } @@ -509,14 +484,15 @@ public class ApplicationController { /** * Deletes the application with this id * - * @return the deleted application, or null if it did not exist * @throws IllegalArgumentException if the application has deployments or the caller is not authorized + * @throws NotExistsException if the application does not exist */ - public Application deleteApplication(ApplicationId id, Optional<NToken> token) { - try (Lock lock = lock(id)) { - Optional<Application> application = get(id); - if ( ! application.isPresent()) return null; - if ( ! application.get().deployments().isEmpty()) + public void deleteApplication(ApplicationId id, Optional<NToken> token) { + if ( ! controller.applications().get(id).isPresent()) + throw new NotExistsException("Could not delete application '" + id + "': Application not found"); + + lockedOrThrow(id, application -> { + if ( ! application.deployments().isEmpty()) throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments"); Tenant tenant = controller.tenants().tenant(new TenantId(id.tenant().value())).get(); @@ -529,9 +505,8 @@ public class ApplicationController { .deleteApplication(tenant.getAthensDomain().get(), new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); db.deleteApplication(id); - log.info("Deleted " + application.get()); - return application.get(); - } + log.info("Deleted " + application); + }); } /** @@ -543,6 +518,30 @@ public class ApplicationController { db.store(application); } + /** + * Acquire a locked application to modify and store, if there is an application with the given id. + * + * @param applicationId Id of the application to lock and get. + * @param actions Things to do with the locked application. + */ + public void lockedIfPresent(ApplicationId applicationId, Consumer<LockedApplication> actions) { + try (Lock lock = lock(applicationId)) { + get(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(actions); + } + } + + /** + * Acquire a locked application to modify and store, or throw an exception if no application has the given id. + * + * @param applicationId Id of the application to lock and require. + * @param actions Things to do with the locked application. + */ + public void lockedOrThrow(ApplicationId applicationId, Consumer<LockedApplication> actions) { + try (Lock lock = lock(applicationId)) { + actions.accept(new LockedApplication(require(applicationId), lock)); + } + } + public void notifyJobCompletion(JobReport report) { if ( ! get(report.applicationId()).isPresent()) { log.log(Level.WARNING, "Ignoring completion of job of project '" + report.projectId() + @@ -582,20 +581,17 @@ public class ApplicationController { private void deactivate(Application application, Zone zone, Optional<Deployment> deployment, boolean requireThatDeploymentHasExpired) { - try (Lock lock = lock(application.id())) { - LockedApplication lockedApplication = controller.applications().require(application.id(), lock); - if (deployment.isPresent() && requireThatDeploymentHasExpired && - ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment.get(), clock.instant())) { - return; - } - lockedApplication = deactivate(lockedApplication, zone); - store(lockedApplication); - } + if (requireThatDeploymentHasExpired && deployment.isPresent() + && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment.get(), clock.instant())) + return; + + lockedOrThrow(application.id(), lockedApplication -> + store(deactivate(lockedApplication, zone))); } - /** + /** * Deactivates a locked application without storing it - * + * * @return the application with the deployment in the given zone removed */ private LockedApplication deactivate(LockedApplication application, Zone zone) { @@ -611,19 +607,19 @@ public class ApplicationController { public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; } private ApplicationId dashToUnderscore(ApplicationId id) { - return ApplicationId.from(id.tenant().value(), + return ApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_"), id.instance().value()); } - + public ConfigServerClient configserverClient() { return configserverClient; } - - /** + + /** * Returns a lock which provides exclusive rights to changing this application. * Any operation which stores an application need to first acquire this lock, then read, modify * and store the application, and finally release (close) the lock. */ - public Lock lock(ApplicationId application) { + Lock lock(ApplicationId application) { return curator.lock(application, Duration.ofMinutes(10)); } @@ -634,19 +630,30 @@ public class ApplicationController { zone.environment().isManuallyDeployed(); } + /** Verify that each of the production zones listed in the deployment spec exist in this system. */ + public void validate(DeploymentSpec deploymentSpec) { + deploymentSpec.zones().stream() + .filter(zone -> zone.environment() == Environment.prod) + .forEach(zone -> { + if ( ! controller.zoneRegistry().getZone(zone.environment(), zone.region().orElse(null)).isPresent()) + throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); + }); + } + + private static final class ApplicationRotation { - + private final ImmutableSet<String> cnames; private final ImmutableSet<Rotation> rotations; - + public ApplicationRotation(Set<String> cnames, Set<Rotation> rotations) { this.cnames = ImmutableSet.copyOf(cnames); this.rotations = ImmutableSet.copyOf(rotations); } - + public Set<String> cnames() { return cnames; } public Set<Rotation> rotations() { return rotations; } - + } - + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 660013daa62..b854ad3f771 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -156,8 +156,8 @@ public class Controller extends AbstractComponent { public Clock clock() { return clock; } - public URI getElkUri(Environment environment, RegionName region, DeploymentId deploymentId) { - return elkUrl(zoneRegistry.getLogServerUri(environment, region), deploymentId); + public URI getElkUri(DeploymentId deploymentId) { + return elkUrl(zoneRegistry.getLogServerUri(deploymentId.zone().environment(), deploymentId.zone().region()), deploymentId); } public List<URI> getConfigServerUris(Environment environment, RegionName region) { @@ -204,7 +204,7 @@ public class Controller extends AbstractComponent { } // TODO: Model the response properly - // TODO: What is this + // TODO: What is this -- I believe it fetches, and purges, errors from some log server public JsonNode grabLog(DeploymentId deploymentId) { return configServerClient.grabLog(deploymentId); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index b0424282ace..e8c8f8a389c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -3,18 +3,26 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.ClusterInfo; +import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; + import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Objects; import java.util.Optional; /** @@ -22,147 +30,159 @@ import java.util.Optional; * fields. * * @author mpolden + * @author jvenstad */ public class LockedApplication extends Application { - private final Lock lock; + private LockedApplication(Builder builder) { + super(builder.applicationId, builder.deploymentSpec, builder.validationOverrides, + builder.deployments, builder.deploymentJobs, builder.deploying, + builder.hasOutstandingChange, builder.ownershipIssueId, builder.metrics); + } /** - * LockedApplication should be acquired through ApplicationController and never constructed directly + * Used to create a locked application * - * @param application Application instance for which lock has been acquired - * @param lock Unused, but must be held when constructing this + * @param application The application to lock. + * @param lock The lock for the application. */ LockedApplication(Application application, Lock lock) { - super(application.id(), application.deploymentSpec(), application.validationOverrides(), - application.deployments(), application.deploymentJobs(), application.deploying(), - application.hasOutstandingChange()); - this.lock = Objects.requireNonNull(lock, "lock cannot be null"); + this(new Builder(application)); } public LockedApplication withProjectId(long projectId) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), deployments(), - deploymentJobs().withProjectId(projectId), deploying(), - hasOutstandingChange()), lock); + return new LockedApplication(new Builder(this).with(deploymentJobs().withProjectId(projectId))); } - public LockedApplication with(IssueId issueId) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), deployments(), - deploymentJobs().with(issueId), deploying(), - hasOutstandingChange()), lock); + public LockedApplication withDeploymentIssueId(IssueId issueId) { + return new LockedApplication(new Builder(this).with(deploymentJobs().with(issueId))); } - public LockedApplication withJobCompletion(DeploymentJobs.JobReport report, Instant notificationTime, - Controller controller) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), - deployments(), - deploymentJobs().withCompletion(report, notificationTime, - controller), - deploying(), hasOutstandingChange()), lock); + public LockedApplication withJobCompletion(DeploymentJobs.JobReport report, Instant notificationTime, Controller controller) { + return new LockedApplication(new Builder(this).with(deploymentJobs().withCompletion(report, notificationTime, controller))); } - public LockedApplication withJobTriggering(long runId, DeploymentJobs.JobType type, Optional<Change> change, - String reason, Instant triggerTime, Controller controller) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), deployments(), - deploymentJobs().withTriggering(type, - change, - runId, - determineTriggerVersion(type, controller), - determineTriggerRevision(type, controller), - reason, - triggerTime), - deploying(), hasOutstandingChange()), lock); + public LockedApplication withJobTriggering(JobType type, Optional<Change> change, Instant triggerTime, + Version version, Optional<ApplicationRevision> revision, String reason) { + return new LockedApplication(new Builder(this).with(deploymentJobs().withTriggering(type, change, version, revision, reason, triggerTime))); } - public LockedApplication with(Deployment deployment) { - Map<Zone, Deployment> deployments = new LinkedHashMap<>(deployments()); - deployments.put(deployment.zone(), deployment); - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), - deployments, deploymentJobs(), deploying(), - hasOutstandingChange()), lock); + public LockedApplication withNewDeployment(Zone zone, ApplicationRevision revision, Version version, Instant instant) { + // Use info from previous deployment if available, otherwise create a new one. + Deployment previousDeployment = deployments().getOrDefault(zone, new Deployment(zone, revision, version, instant)); + Deployment newDeployment = new Deployment(zone, revision, version, instant, + previousDeployment.clusterUtils(), + previousDeployment.clusterInfo(), + previousDeployment.metrics()); + return with(newDeployment); + } + + public LockedApplication withClusterUtilization(Zone zone, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) { + Deployment deployment = deployments().get(zone); + if (deployment == null) return this; // No longer deployed in this zone. + return with(deployment.withClusterUtils(clusterUtilization)); } - public LockedApplication with(DeploymentJobs deploymentJobs) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), - deployments(), deploymentJobs, deploying(), - hasOutstandingChange()), lock); + public LockedApplication withClusterInfo(Zone zone, Map<ClusterSpec.Id, ClusterInfo> clusterInfo) { + Deployment deployment = deployments().get(zone); + if (deployment == null) return this; // No longer deployed in this zone. + return with(deployment.withClusterInfo(clusterInfo)); + + } + + public LockedApplication with(Zone zone, DeploymentMetrics deploymentMetrics) { + Deployment deployment = deployments().get(zone); + if (deployment == null) return this; // No longer deployed in this zone. + return with(deployment.withMetrics(deploymentMetrics)); } public LockedApplication withoutDeploymentIn(Zone zone) { Map<Zone, Deployment> deployments = new LinkedHashMap<>(deployments()); deployments.remove(zone); - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), - deployments, deploymentJobs(), deploying(), - hasOutstandingChange()), lock); + return new LockedApplication(new Builder(this).with(deployments)); } public LockedApplication withoutDeploymentJob(DeploymentJobs.JobType jobType) { - DeploymentJobs deploymentJobs = deploymentJobs().without(jobType); - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), - deployments(), deploymentJobs, deploying(), - hasOutstandingChange()), lock); + return new LockedApplication(new Builder(this).with(deploymentJobs().without(jobType))); } public LockedApplication with(DeploymentSpec deploymentSpec) { - return new LockedApplication(new Application(id(), deploymentSpec, validationOverrides(), - deployments(), deploymentJobs(), deploying(), - hasOutstandingChange()), lock); + return new LockedApplication(new Builder(this).with(deploymentSpec)); } public LockedApplication with(ValidationOverrides validationOverrides) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides, - deployments(), deploymentJobs(), deploying(), - hasOutstandingChange()), lock); + return new LockedApplication(new Builder(this).with(validationOverrides)); } public LockedApplication withDeploying(Optional<Change> deploying) { - return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), - deployments(), deploymentJobs(), deploying, - hasOutstandingChange()), lock); + return new LockedApplication(new Builder(this).withDeploying(deploying)); } public LockedApplication withOutstandingChange(boolean outstandingChange) { - return new LockedApplication(new Application(id(), deploymentSpec(), - validationOverrides(), deployments(), - deploymentJobs(), deploying(), outstandingChange), lock); - } - - private Version determineTriggerVersion(DeploymentJobs.JobType jobType, Controller controller) { - Optional<Zone> zone = jobType.zone(controller.system()); - if ( ! zone.isPresent()) // a sloppy test TODO: Fix - return controller.systemVersion(); - return currentDeployVersion(controller, zone.get()); - } - - private Optional<ApplicationRevision> determineTriggerRevision(DeploymentJobs.JobType jobType, - Controller controller) { - Optional<Zone> zone = jobType.zone(controller.system()); - if ( ! zone.isPresent()) // a sloppy test TODO: Fix - return Optional.empty(); - return currentDeployRevision(jobType.zone(controller.system()).get()); - } - - /** Returns the version a deployment to this zone should use for this application, or empty if we don't know */ - private Optional<ApplicationRevision> currentDeployRevision(Zone zone) { - if (!deploying().isPresent()) { - return currentRevision(zone); - } else if (deploying().get() instanceof Change.VersionChange) { - return currentRevision(zone); - } else { - return ((Change.ApplicationChange) deploying().get()).revision(); - } + return new LockedApplication(new Builder(this).with(outstandingChange)); } - /** - * Returns the current revision this application has, or if none; should use assuming no change, - * in the given zone. Empty if not known - */ - private Optional<ApplicationRevision> currentRevision(Zone zone) { - Deployment currentDeployment = deployments().get(zone); - if (currentDeployment != null) { // Already deployed in this zone: Use that revision - return Optional.of(currentDeployment.revision()); + public LockedApplication withOwnershipIssueId(IssueId issueId) { + return new LockedApplication(new Builder(this).withOwnershipIssueId(Optional.ofNullable(issueId))); + } + + public LockedApplication with(MetricsService.ApplicationMetrics metrics) { + return new LockedApplication(new Builder(this).with(metrics)); + } + + public Version deployVersionFor(DeploymentJobs.JobType jobType, Controller controller) { + return jobType == JobType.component + ? controller.systemVersion() + : deployVersionIn(jobType.zone(controller.system()).get(), controller); + } + + public Optional<ApplicationRevision> deployRevisionFor(DeploymentJobs.JobType jobType, Controller controller) { + return jobType == JobType.component + ? Optional.empty() + : deployRevisionIn(jobType.zone(controller.system()).get()); + } + + /** Don't expose non-leaf sub-objects. */ + private LockedApplication with(Deployment deployment) { + Map<Zone, Deployment> deployments = new LinkedHashMap<>(deployments()); + deployments.put(deployment.zone(), deployment); + return new LockedApplication(new Builder(this).with(deployments)); + } + + + private static class Builder { + + private final ApplicationId applicationId; + private DeploymentSpec deploymentSpec; + private ValidationOverrides validationOverrides; + private Map<Zone, Deployment> deployments; + private DeploymentJobs deploymentJobs; + private Optional<Change> deploying; + private boolean hasOutstandingChange; + private Optional<IssueId> ownershipIssueId; + private ApplicationMetrics metrics; + + private Builder(Application application) { + this.applicationId = application.id(); + this.deploymentSpec = application.deploymentSpec(); + this.validationOverrides = application.validationOverrides(); + this.deployments = application.deployments(); + this.deploymentJobs = application.deploymentJobs(); + this.deploying = application.deploying(); + this.hasOutstandingChange = application.hasOutstandingChange(); + this.ownershipIssueId = application.ownershipIssueId(); + this.metrics = application.metrics(); } - return Optional.empty(); + + private Builder with(DeploymentSpec deploymentSpec) { this.deploymentSpec = deploymentSpec; return this; } + private Builder with(ValidationOverrides validationOverrides) { this.validationOverrides = validationOverrides; return this; } + private Builder with(Map<Zone, Deployment> deployments) { this.deployments = deployments; return this; } + private Builder with(DeploymentJobs deploymentJobs) { this.deploymentJobs = deploymentJobs; return this; } + private Builder withDeploying(Optional<Change> deploying) { this.deploying = deploying; return this; } + private Builder with(boolean hasOutstandingChange) { this.hasOutstandingChange = hasOutstandingChange; return this; } + private Builder withOwnershipIssueId(Optional<IssueId> ownershipIssueId) { this.ownershipIssueId = ownershipIssueId; return this; } + private Builder with(ApplicationMetrics metrics) { this.metrics = metrics; return this; } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 710d2ad6492..4d1a009806f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import java.time.Instant; +import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -23,17 +24,17 @@ public class ApplicationList { private final ImmutableList<Application> list; - private ApplicationList(List<Application> applications) { + private ApplicationList(Iterable<Application> applications) { this.list = ImmutableList.copyOf(applications); } // ----------------------------------- Factories - public static ApplicationList from(List<Application> applications) { + public static ApplicationList from(Iterable<Application> applications) { return new ApplicationList(applications); } - public static ApplicationList from(List<ApplicationId> ids, ApplicationController applications) { + public static ApplicationList from(Collection<ApplicationId> ids, ApplicationController applications) { return listOf(ids.stream().map(applications::require)); } @@ -42,6 +43,9 @@ public class ApplicationList { /** Returns the applications in this as an immutable list */ public List<Application> asList() { return list; } + /** Returns the ids of the applications in this as an immutable list */ + public List<ApplicationId> idList() { return ImmutableList.copyOf(list.stream().map(Application::id)::iterator); } + public boolean isEmpty() { return list.isEmpty(); } public int size() { return list.size(); } @@ -82,11 +86,21 @@ public class ApplicationList { return listOf(list.stream().filter(application -> ! isDeployingApplicationChange(application))); } + /** Returns the subset of applications which is currently not deploying a change */ + public ApplicationList notDeploying() { + return listOf(list.stream().filter(application -> ! application.deploying().isPresent())); + } + /** Returns the subset of applications which currently does not have any failing jobs */ public ApplicationList notFailing() { return listOf(list.stream().filter(application -> ! application.deploymentJobs().hasFailures())); } + /** Returns the subset of applications which currently have failing jobs */ + public ApplicationList failing() { + return listOf(list.stream().filter(application -> application.deploymentJobs().hasFailures())); + } + /** Returns the subset of applications which have been failing an upgrade to the given version since the given instant */ public ApplicationList failingUpgradeToVersionSince(Version version, Instant threshold) { return listOf(list.stream().filter(application -> failingUpgradeToVersionSince(application, version, threshold))); @@ -102,14 +116,14 @@ public class ApplicationList { return listOf(list.stream().filter(application -> ! failingOn(version, application))); } - /** Returns the subset of applications which have at least one deployment */ + /** Returns the subset of applications which have at least one production deployment */ public ApplicationList hasDeployment() { return listOf(list.stream().filter(a -> !a.productionDeployments().isEmpty())); } /** Returns the subset of applications which started failing after the given instant */ - public ApplicationList startedFailingAfter(Instant instant) { - return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant))); + public ApplicationList startedFailingOnVersionAfter(Version version, Instant instant) { + return listOf(list.stream().filter(application -> JobList.from(application).firstFailing().on(version).firstFailing().after(instant).anyMatch())); } /** Returns the subset of applications which has the given upgrade policy */ @@ -161,12 +175,12 @@ public class ApplicationList { * Applications without any deployments are ordered first. */ public ApplicationList byIncreasingDeployedVersion() { - return listOf(list.stream().sorted(Comparator.comparing(application -> application.deployedVersion().orElse(Version.emptyVersion)))); + return listOf(list.stream().sorted(Comparator.comparing(application -> application.oldestDeployedVersion().orElse(Version.emptyVersion)))); } /** Returns the subset of applications that are not currently upgrading */ public ApplicationList notCurrentlyUpgrading(Change.VersionChange change, Instant jobTimeoutLimit) { - return listOf(list.stream().filter(a -> !currentlyUpgrading(change, a, jobTimeoutLimit))); + return listOf(list.stream().filter(a -> ! currentlyUpgrading(change, a, jobTimeoutLimit))); } // ----------------------------------- Internal helpers @@ -193,56 +207,39 @@ public class ApplicationList { if ( ! application.deploying().isPresent()) return false; return application.deploying().get() instanceof Change.ApplicationChange; } - + private static boolean failingOn(Version version, Application application) { - for (JobStatus jobStatus : application.deploymentJobs().jobStatus().values()) - if ( ! jobStatus.isSuccess() && jobStatus.lastCompleted().get().version().equals(version)) return true; - return false; + return JobList.from(application) + .failing() + .lastCompleted().on(version) + .anyMatch(); } private static boolean currentlyUpgrading(Change.VersionChange change, Application application, Instant jobTimeoutLimit) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(status -> status.isRunning(jobTimeoutLimit)) - .filter(status -> status.lastTriggered().isPresent()) - .map(status -> status.lastTriggered().get()) - .anyMatch(jobRun -> jobRun.version().equals(change.version())); + return JobList.from(application) + .running(jobTimeoutLimit) + .lastTriggered().on(change.version()) + .anyMatch(); } private static boolean failingUpgradeToVersionSince(Application application, Version version, Instant threshold) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(job -> isUpgradeFailure(job)) - .filter(job -> job.firstFailing().get().at().isBefore(threshold)) - .anyMatch(job -> job.lastCompleted().get().version().equals(version)); + return JobList.from(application) + .not().failingApplicationChange() + .firstFailing().before(threshold) + .lastCompleted().on(version) + .anyMatch(); } private static boolean failingApplicationChangeSince(Application application, Instant threshold) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(job -> isApplicationChangeFailure(job)) - .anyMatch(job -> job.firstFailing().get().at().isBefore(threshold)); - } - - private static boolean isUpgradeFailure(JobStatus job) { - if ( job.isSuccess()) return false; - if ( ! job.lastSuccess().isPresent()) return false; // An application which never succeeded is surely bad. - if ( ! job.lastSuccess().get().revision().isPresent()) return false; // Indicates the component job, which is not an upgrade. - if ( ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision())) return false; // Application change may be to blame. - return ! job.firstFailing().get().version().equals(job.lastSuccess().get().version()); // Return whether there is a version change. + return JobList.from(application) + .failingApplicationChange() + .firstFailing().before(threshold) + .anyMatch(); } - private static boolean isApplicationChangeFailure(JobStatus job) { - if ( job.isSuccess()) return false; - if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad. - if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change. - if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame. - return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change. - } - - /** Convenience converter from a stream to an ApplicationList */ private static ApplicationList listOf(Stream<Application> applications) { - ImmutableList.Builder<Application> b = new ImmutableList.Builder<>(); - applications.forEach(b::add); - return new ApplicationList(b.build()); + return from(applications::iterator); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index bf067a39bbb..d9c22018d26 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -33,7 +33,7 @@ public abstract class Change { @Override public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) { - return ! deploymentSpec.canChangeRevisionAt(instant); + return ! deploymentSpec.canChangeRevisionAt(instant); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 8f654c66871..98f8c2a3d99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -13,12 +13,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import java.time.Instant; import java.util.Collection; -import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Information about which deployment jobs an application should run and their current status. @@ -66,7 +68,6 @@ public class DeploymentJobs { public DeploymentJobs withTriggering(JobType jobType, Optional<Change> change, - long runId, Version version, Optional<ApplicationRevision> revision, String reason, @@ -74,8 +75,7 @@ public class DeploymentJobs { Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); status.compute(jobType, (type, job) -> { if (job == null) job = JobStatus.initial(jobType); - return job.withTriggering(runId, - version, + return job.withTriggering( version, revision, change.isPresent() && change.get() instanceof Change.VersionChange, reason, @@ -103,12 +103,12 @@ public class DeploymentJobs { /** Returns whether this has some job status which is not a success */ public boolean hasFailures() { - return status.values().stream().anyMatch(jobStatus -> ! jobStatus.isSuccess()); + return JobList.from(status.values()).failing().anyMatch(); } /** Returns whether any job is currently in progress */ public boolean isRunning(Instant timeoutLimit) { - return status.values().stream().anyMatch(job -> job.isRunning(timeoutLimit)); + return JobList.from(status.values()).running(timeoutLimit).anyMatch(); } /** Returns whether the given job type is currently running and was started after timeoutLimit */ @@ -120,7 +120,7 @@ public class DeploymentJobs { /** Returns whether change can be deployed to the given environment */ public boolean isDeployableTo(Environment environment, Optional<Change> change) { - if (environment == null || !change.isPresent()) { + if (environment == null || ! change.isPresent()) { return true; } if (environment == Environment.staging) { @@ -131,31 +131,13 @@ public class DeploymentJobs { return true; // other environments do not have any preconditions } - /** Returns whether the given change has been deployed completely */ - public boolean isDeployed(Change change) { - return status.values().stream() - .filter(status -> status.type().isProduction()) - .allMatch(status -> isSuccessful(change, status.type())); - } - /** Returns whether job has completed successfully */ public boolean isSuccessful(Change change, JobType jobType) { return Optional.ofNullable(jobStatus().get(jobType)) - .filter(JobStatus::isSuccess) - .filter(status -> status.lastCompletedFor(change)) + .flatMap(JobStatus::lastSuccess) + .filter(status -> status.lastCompletedWas(change)) .isPresent(); } - - /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */ - public Instant failingSince() { - Instant failingSince = null; - for (JobStatus jobStatus : jobStatus().values()) { - if (jobStatus.isSuccess()) continue; - if (failingSince == null || failingSince.isAfter(jobStatus.firstFailing().get().at())) - failingSince = jobStatus.firstFailing().get().at(); - } - return failingSince; - } /** * Returns the id of the Screwdriver project running these deployment jobs @@ -166,39 +148,44 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } + private static Optional<Long> requireId(Optional<Long> id, String message) { + Objects.requireNonNull(id, message); + if ( ! id.isPresent()) { + return id; + } + if (id.get() <= 0) { + throw new IllegalArgumentException(message); + } + return id; + } + /** Job types that exist in the build system */ public enum JobType { - component("component"), - systemTest("system-test", zone(SystemName.cd, "test", "cd-us-central-1"), zone("test", "us-east-1")), - stagingTest("staging-test", zone(SystemName.cd, "staging", "cd-us-central-1"), zone("staging", "us-east-3")), - productionCorpUsEast1("production-corp-us-east-1", zone("prod", "corp-us-east-1")), - productionUsEast3("production-us-east-3", zone("prod", "us-east-3")), - productionUsWest1("production-us-west-1", zone("prod", "us-west-1")), - productionUsCentral1("production-us-central-1", zone("prod", "us-central-1")), - productionApNortheast1("production-ap-northeast-1", zone("prod", "ap-northeast-1")), - productionApNortheast2("production-ap-northeast-2", zone("prod", "ap-northeast-2")), - productionApSoutheast1("production-ap-southeast-1", zone("prod", "ap-southeast-1")), - productionEuWest1("production-eu-west-1", zone("prod", "eu-west-1")), - productionCdUsCentral1("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")), - productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); - - private final String id; - private final Map<SystemName, Zone> zones; - - JobType(String id, Zone... zone) { - this.id = id; - Map<SystemName, Zone> zones = new HashMap<>(); - for (Zone z : zone) { - if (zones.containsKey(z.system())) { - throw new IllegalArgumentException("A job can only map to a single zone per system"); - } - zones.put(z.system(), z); - } - this.zones = Collections.unmodifiableMap(zones); + component ("component" ), + systemTest ("system-test" , zone("test" , "us-east-1" ), zone(SystemName.cd, "test" , "cd-us-central-1")), + stagingTest ("staging-test" , zone("staging", "us-east-3" ), zone(SystemName.cd, "staging", "cd-us-central-1")), + productionCorpUsEast1 ("production-corp-us-east-1" , zone("prod" , "corp-us-east-1")), + productionUsEast3 ("production-us-east-3" , zone("prod" , "us-east-3" )), + productionUsWest1 ("production-us-west-1" , zone("prod" , "us-west-1" )), + productionUsCentral1 ("production-us-central-1" , zone("prod" , "us-central-1" )), + productionApNortheast1 ("production-ap-northeast-1" , zone("prod" , "ap-northeast-1")), + productionApNortheast2 ("production-ap-northeast-2" , zone("prod" , "ap-northeast-2")), + productionApSoutheast1 ("production-ap-southeast-1" , zone("prod" , "ap-southeast-1")), + productionEuWest1 ("production-eu-west-1" , zone("prod" , "eu-west-1" )), + productionCdUsCentral1 ("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")), + productionCdUsCentral2 ("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); + + private final String jobName; + private final ImmutableMap<SystemName, Zone> zones; + + JobType(String jobName, Zone... zones) { + this.jobName = jobName; + this.zones = ImmutableMap.copyOf(Stream.of(zones).collect(Collectors.toMap(zone -> zone.system(), + zone -> zone))); } - public String id() { return id; } + public String jobName() { return jobName; } /** Returns the zone for this job in the given system, or empty if this job does not have a zone */ public Optional<Zone> zone(SystemName system) { @@ -223,42 +210,26 @@ public class DeploymentJobs { return zone(system).map(Zone::region); } - public static JobType fromId(String id) { - switch (id) { - case "component" : return component; - case "system-test" : return systemTest; - case "staging-test" : return stagingTest; - case "production-corp-us-east-1" : return productionCorpUsEast1; - case "production-us-east-3" : return productionUsEast3; - case "production-us-west-1" : return productionUsWest1; - case "production-us-central-1" : return productionUsCentral1; - case "production-ap-northeast-1" : return productionApNortheast1; - case "production-ap-northeast-2" : return productionApNortheast2; - case "production-ap-southeast-1" : return productionApSoutheast1; - case "production-eu-west-1" : return productionEuWest1; - case "production-cd-us-central-1" : return productionCdUsCentral1; - case "production-cd-us-central-2" : return productionCdUsCentral2; - default : throw new IllegalArgumentException("Unknown job id '" + id + "'"); - } + public static JobType fromJobName(String jobName) { + return Stream.of(values()) + .filter(jobType -> jobType.jobName.equals(jobName)) + .findAny().orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); } - /** Returns the job type for the given zone, or null if none */ - public static JobType from(SystemName system, com.yahoo.config.provision.Zone zone) { - for (JobType job : values()) { - Optional<com.yahoo.config.provision.Zone> jobZone = job.zone(system); - if (jobZone.isPresent() && jobZone.get().equals(zone)) - return job; - } - return null; + /** Returns the job type for the given zone */ + public static Optional<JobType> from(SystemName system, Zone zone) { + return Stream.of(values()) + .filter(job -> job.zone(system).filter(zone::equals).isPresent()) + .findAny(); } /** Returns the job job type for the given environment and region or null if none */ - public static JobType from(SystemName system, Environment environment, RegionName region) { + public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) { switch (environment) { - case test: return systemTest; - case staging: return stagingTest; + case test: return Optional.of(systemTest); + case staging: return Optional.of(stagingTest); } - return from(system, new com.yahoo.config.provision.Zone(environment, region)); + return from(system, new Zone(environment, region)); } private static Zone zone(SystemName system, String environment, String region) { @@ -296,7 +267,7 @@ public class DeploymentJobs { public JobType jobType() { return jobType; } public long projectId() { return projectId; } public long buildNumber() { return buildNumber; } - public boolean success() { return !jobError.isPresent(); } + public boolean success() { return ! jobError.isPresent(); } public Optional<JobError> jobError() { return jobError; } } @@ -304,23 +275,6 @@ public class DeploymentJobs { public enum JobError { unknown, outOfCapacity; - - public static Optional<JobError> from(boolean success) { - return Optional.of(success) - .filter(b -> !b) - .map(ignored -> unknown); - } - } - - private static Optional<Long> requireId(Optional<Long> id, String message) { - Objects.requireNonNull(id, message); - if (!id.isPresent()) { - return id; - } - if (id.get() <= 0) { - throw new IllegalArgumentException(message); - } - return id; } }
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java index 504e2285a34..c0f7bd6c6a1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; /** * @author smorgrav diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java new file mode 100644 index 00000000000..6223b07d27a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -0,0 +1,189 @@ +package com.yahoo.vespa.hosted.controller.application; + +import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; + +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; + +/** + * A list of deployment jobs that can be filtered in various ways. + * + * @author jvenstad + */ +public class JobList { + + private final ImmutableList<JobStatus> list; + private final boolean negate; + + private JobList(Iterable<JobStatus> jobs, boolean negate) { + this.list = ImmutableList.copyOf(jobs); + this.negate = negate; + } + + private JobList(Iterable<JobStatus> jobs) { + this(jobs, false); + } + + // ----------------------------------- Factories + + public static JobList from(Iterable<JobStatus> jobs) { + return new JobList(jobs); + } + + public static JobList from(Application application) { + return from(application.deploymentJobs().jobStatus().values()); + } + + // ----------------------------------- Accessors + + // TODO: Add sorting based on various stuff, such as deployment order, time of last completion, etc.. + + /** Returns the jobstatuses in this as an immutable list */ + public List<JobStatus> asList() { return list; } + + /** Returns the jobstatuses in this as an immutable list after mapping with the given function */ + public <Type> List<Type> mapToList(Function<JobStatus, Type> mapper) { + return ImmutableList.copyOf(list.stream().map(mapper)::iterator); + } + + public boolean isEmpty() { return list.isEmpty(); } + + public boolean anyMatch() { return ! isEmpty(); } + + public int size() { return list.size(); } + + // ----------------------------------- Basic filters + + /** Negates the next filter operation */ + public JobList not() { + return new JobList(list, ! negate); + } + + /** Returns the subset of jobs which are current upgrading */ + public JobList upgrading() { // TODO: Centralise and standardise reasoning about upgrades and revisions. + return filter(job -> job.lastSuccess().isPresent() + && job.lastTriggered().isPresent() + && ! job.lastTriggered().get().at().isBefore(job.lastCompleted().get().at()) + && job.lastSuccess().get().version().isBefore(job.lastTriggered().get().version())); + } + + /** Returns the subset of jobs which are currently running, according to the given timeout */ + public JobList running(Instant timeoutLimit) { + return filter(job -> job.isRunning(timeoutLimit)); + } + + /** Returns the subset of jobs which are currently failing */ + public JobList failing() { + return filter(job -> ! job.isSuccess()); + } + + /** Returns the subset of jobs which must be failing due to an application change */ + public JobList failingApplicationChange() { + return filter(job -> failingApplicationChange(job)); + } + + /** Returns the subset of jobs which are failing with the given job error */ + public JobList failingBecause(DeploymentJobs.JobError error) { + return filter(job -> job.jobError().filter(error::equals).isPresent()); + } + + /** Returns the subset of jobs of the given type -- most useful when negated */ + public JobList type(JobType type) { + return filter(job -> job.type() == type); + } + + /** Returns the subset of jobs of which are production jobs */ + public JobList production() { + return filter(job -> job.type().isProduction()); + } + + // ----------------------------------- JobRun filtering + + /** Returns the list in a state where the next filter is for the lastTriggered run type */ + public JobRunFilter lastTriggered() { + return new JobRunFilter(job -> job.lastTriggered()); + } + + /** Returns the list in a state where the next filter is for the lastCompleted run type */ + public JobRunFilter lastCompleted() { + return new JobRunFilter(job -> job.lastCompleted()); + } + + /** Returns the list in a state where the next filter is for the lastSuccess run type */ + public JobRunFilter lastSuccess() { + return new JobRunFilter(job -> job.lastSuccess()); + } + + /** Returns the list in a state where the next filter is for the firstFailing run type */ + public JobRunFilter firstFailing() { + return new JobRunFilter(job -> job.firstFailing()); + } + + + /** Allows sub-filters for runs of the given kind */ + public class JobRunFilter { + + private final Function<JobStatus, Optional<JobRun>> which; + + private JobRunFilter(Function<JobStatus, Optional<JobRun>> which) { + this.which = which; + } + + /** Returns the subset of jobs where the run of the given type exists */ + public JobList present() { + return filter(run -> true); + } + + /** Returns the subset of jobs where the run of the given type occurred before the given instant */ + public JobList before(Instant threshold) { + return filter(run -> run.at().isBefore(threshold)); + } + + /** Returns the subset of jobs where the run of the given type occurred after the given instant */ + public JobList after(Instant threshold) { + return filter(run -> run.at().isAfter(threshold)); + } + + /** Returns the subset of jobs where the run of the given type was on the given version */ + public JobList on(Version version) { + return filter(run -> run.version().equals(version)); + } + + public JobList upgrade() { + return filter(run -> run.upgrade()); + } + + /** Transforms the JobRun condition to a JobStatus condition, by considering only the JobRun mapped by which, and executes */ + private JobList filter(Predicate<JobRun> condition) { + return JobList.this.filter(job -> which.apply(job).filter(condition).isPresent()); + } + + } + + + // ----------------------------------- Internal helpers + + private static boolean failingApplicationChange(JobStatus job) { + if ( job.isSuccess()) return false; + if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad. + if ( ! job.lastSuccess().get().revision().isPresent()) return true; // Indicates the component job, which is always an application change. + if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame. + return ! job.firstFailing().get().revision().equals(job.lastSuccess().get().revision()); // Return whether there is an application change. + } + + /** Returns a new JobList which is the result of filtering with the -- possibly negated -- condition */ + private JobList filter(Predicate<JobStatus> condition) { + return from(list.stream().filter(negate ? condition.negate() : condition)::iterator); + } + +} + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index 929bf186eea..ceb04d88026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Controller; -import java.time.Duration; import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -56,9 +55,9 @@ public class JobStatus { return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } - public JobStatus withTriggering(long runId, Version version, Optional<ApplicationRevision> revision, + public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, boolean upgrade, String reason, Instant triggerTime) { - return new JobStatus(type, jobError, Optional.of(new JobRun(runId, version, revision, upgrade, reason, triggerTime)), + return new JobStatus(type, jobError, Optional.of(new JobRun(-1, version, revision, upgrade, reason, triggerTime)), lastCompleted, firstFailing, lastSuccess); } @@ -75,7 +74,7 @@ public class JobStatus { } else if (! lastTriggered.isPresent()) { throw new IllegalStateException("Got notified about completion of " + this + - ", but that has not been triggered nor deployed"); + ", but that has neither been triggered nor deployed"); } else { @@ -103,14 +102,16 @@ public class JobStatus { public DeploymentJobs.JobType type() { return type; } /** Returns true unless this job last completed with a failure */ - public boolean isSuccess() { return ! jobError.isPresent(); } + public boolean isSuccess() { + return lastCompleted().isPresent() && ! jobError.isPresent(); + } /** Returns true if last triggered is newer than last completed and was started after timeoutLimit */ public boolean isRunning(Instant timeoutLimit) { if ( ! lastTriggered.isPresent()) return false; if (lastTriggered.get().at().isBefore(timeoutLimit)) return false; if ( ! lastCompleted.isPresent()) return true; - return lastTriggered.get().at().isAfter(lastCompleted.get().at()); + return ! lastTriggered.get().at().isBefore(lastCompleted.get().at()); } /** The error of the last completion, or empty if the last run succeeded */ @@ -131,18 +132,6 @@ public class JobStatus { /** Returns the run when this last succeeded, or empty if it has never succeeded */ public Optional<JobRun> lastSuccess() { return lastSuccess; } - /** Returns whether the job last completed for the given change */ - public boolean lastCompletedFor(Change change) { - if (change instanceof Change.ApplicationChange) { - Change.ApplicationChange applicationChange = (Change.ApplicationChange) change; - return lastCompleted().isPresent() && lastCompleted().get().revision().equals(applicationChange.revision()); - } else if (change instanceof Change.VersionChange) { - Change.VersionChange versionChange = (Change.VersionChange) change; - return lastCompleted().isPresent() && lastCompleted().get().version().equals(versionChange.version()); - } - throw new IllegalArgumentException("Unexpected change: " + change.getClass()); - } - @Override public String toString() { return "job status of " + type + "[ " + @@ -191,10 +180,12 @@ public class JobStatus { this.reason = reason; this.at = at; } - + + // TODO: Replace with proper ID, and make the build number part optional, or something -- it's not there for lastTriggered! /** Returns the id of this run of this job, or -1 if not known */ public long id() { return id; } + // TODO: Fix how this is set, and add an applicationChange() method as well, in the same vein. /** Returns whether this job run was a Vespa upgrade */ public boolean upgrade() { return upgrade; } @@ -210,6 +201,19 @@ public class JobStatus { /** Returns the time if this triggering or completion */ public Instant at() { return at; } + // TODO: Consider a version and revision for each JobStatus, to compare against a Target (instead of Change, which is, really, a Target). + /** Returns whether the job last completed for the given change */ + public boolean lastCompletedWas(Change change) { + if (change instanceof Change.ApplicationChange) { + Change.ApplicationChange applicationChange = (Change.ApplicationChange) change; + return revision().equals(applicationChange.revision()); + } else if (change instanceof Change.VersionChange) { + Change.VersionChange versionChange = (Change.VersionChange) change; + return version().equals(versionChange.version()); + } + throw new IllegalArgumentException("Unexpected change: " + change.getClass()); + } + @Override public int hashCode() { return Objects.hash(version, revision, upgrade, at); @@ -220,7 +224,7 @@ public class JobStatus { if (this == o) return true; if ( ! (o instanceof JobRun)) return false; JobRun jobRun = (JobRun) o; - return id == id && + return id == jobRun.id && Objects.equals(version, jobRun.version) && Objects.equals(revision, jobRun.revision) && upgrade == jobRun.upgrade && diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java index de771ff2e17..51865be04fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java @@ -3,29 +3,27 @@ package com.yahoo.vespa.hosted.controller.athenz.filter; import com.google.inject.Inject; import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.handler.FastContentWriter; -import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.jdisc.http.server.jetty.ErrorResponseContentCreator; import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; import com.yahoo.vespa.hosted.controller.athenz.NToken; import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; -import java.util.Optional; import java.util.concurrent.Executor; +import static com.yahoo.vespa.hosted.controller.athenz.filter.SecurityFilterUtils.sendErrorResponse; + /** * Performs authentication by validating the principal token (NToken) header. * * @author bjorncs */ +// TODO bjorncs: Move this class into separate container-security bundle public class AthenzPrincipalFilter implements SecurityRequestFilter { - private final ErrorResponseContentCreator responseCreator = new ErrorResponseContentCreator(); private final NTokenValidator validator; private final String principalTokenHeader; @@ -47,7 +45,7 @@ public class AthenzPrincipalFilter implements SecurityRequestFilter { public void filter(DiscFilterRequest request, ResponseHandler responseHandler) { String rawToken = request.getHeader(principalTokenHeader); if (rawToken == null || rawToken.isEmpty()) { - sendUnauthorized(request, responseHandler, "NToken is missing"); + sendErrorResponse(responseHandler, Response.Status.UNAUTHORIZED, "NToken is missing"); return; } try { @@ -55,16 +53,7 @@ public class AthenzPrincipalFilter implements SecurityRequestFilter { request.setUserPrincipal(principal); request.setRemoteUser(principal.getName()); } catch (InvalidTokenException e) { - sendUnauthorized(request, responseHandler, e.getMessage()); - } - } - - private void sendUnauthorized(DiscFilterRequest request, ResponseHandler responseHandler, String message) { - try (FastContentWriter writer = ResponseDispatch.newInstance(Response.Status.UNAUTHORIZED) - .connectFastWriter(responseHandler)) { - writer.write( - responseCreator.createErrorContent( - request.getRequestURI(), Response.Status.UNAUTHORIZED, Optional.of(message))); + sendErrorResponse(responseHandler,Response.Status.UNAUTHORIZED, e.getMessage()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java new file mode 100644 index 00000000000..8e193d3848f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/SecurityFilterUtils.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.FastContentWriter; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; + +/** + * @author bjorncs + */ +class SecurityFilterUtils { + private static final ObjectMapper mapper = new ObjectMapper(); + + private SecurityFilterUtils() {} + + static void sendErrorResponse(ResponseHandler responseHandler, int statusCode, String message) { + Response response = new Response(statusCode); + response.headers().put("Content-Type", "application/json"); + ObjectNode errorMessage = mapper.createObjectNode(); + errorMessage.put("message", message); + try (FastContentWriter writer = ResponseDispatch.newInstance(response).connectFastWriter(responseHandler)) { + writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorMessage)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java new file mode 100644 index 00000000000..bfa543f160a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java @@ -0,0 +1,106 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; +import com.yahoo.vespa.hosted.controller.restapi.application.Authorizer; + +import java.security.Principal; +import java.util.concurrent.Executor; +import java.util.logging.Logger; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.controller.athenz.filter.SecurityFilterUtils.sendErrorResponse; + +/** + * A variant of the {@link AthenzPrincipalFilter} to be used in combination with a cookie-based + * security filter for user authentication + * Assumes that the user authentication filter configured in the same filter chain and is configured to run before this filter. + * + * @author bjorncs + */ +// TODO Remove this filter once migrated to Okta +public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { + + private static final Logger log = Logger.getLogger(UserAuthWithAthenzPrincipalFilter.class.getName()); + + private final String userAuthenticationPassThruAttribute; + + @Inject + public UserAuthWithAthenzPrincipalFilter(ZmsKeystore zmsKeystore, Executor executor, AthenzConfig config) { + super(zmsKeystore, executor, config); + this.userAuthenticationPassThruAttribute = config.userAuthenticationPassThruAttribute(); + } + + @Override + public void filter(DiscFilterRequest request, ResponseHandler responseHandler) { + if (request.getMethod().equals("OPTIONS")) return; // Skip authentication on OPTIONS - required for Javascript CORS + + try { + switch (getUserAuthenticationResult(request)) { + case USER_COOKIE_MISSING: + case USER_COOKIE_ALTERNATIVE_MISSING: + super.filter(request, responseHandler); // Cookie-based authentication failed, delegate to Athenz + break; + case USER_COOKIE_OK: + rewriteUserPrincipalToAthenz(request); + return; // Authenticated using user cookie + case USER_COOKIE_INVALID: + sendErrorResponse(responseHandler, + Response.Status.UNAUTHORIZED, + "Your user cookie is invalid (either expired, tampered or invalid ip)"); + break; + } + } catch (Exception e) { + log.log(LogLevel.WARNING, "Authentication failed: " + e.getMessage(), e); + sendErrorResponse(responseHandler, Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private UserAuthenticationResult getUserAuthenticationResult(DiscFilterRequest request) { + if (!request.containsAttribute(userAuthenticationPassThruAttribute)) { + throw new IllegalStateException("User authentication filter passthru attribute missing"); + } + Integer statusCode = (Integer) request.getAttribute(userAuthenticationPassThruAttribute); + return Stream.of(UserAuthenticationResult.values()) + .filter(uar -> uar.statusCode == statusCode) + .findAny() + .orElseThrow(() -> new IllegalStateException("Invalid status code: " + statusCode)); + } + + /** + * NOTE: The Bouncer user roles ({@link DiscFilterRequest#roles} are still intact as they are required + * for {@link Authorizer#isMemberOfVespaBouncerGroup(HttpRequest)}. + */ + private static void rewriteUserPrincipalToAthenz(DiscFilterRequest request) { + Principal userPrincipal = request.getUserPrincipal(); + log.log(LogLevel.DEBUG, () -> "Original user principal: " + userPrincipal.toString()); + UserId userId = new UserId(userPrincipal.getName()); + AthenzPrincipal athenzPrincipal = AthenzUtils.createPrincipal(userId); + request.setUserPrincipal(athenzPrincipal); + request.setRemoteUser(athenzPrincipal.toYRN()); + } + + private enum UserAuthenticationResult { + USER_COOKIE_MISSING(0), + USER_COOKIE_OK(1), + USER_COOKIE_INVALID(-1), + USER_COOKIE_ALTERNATIVE_MISSING(-2); + + final int statusCode; + + UserAuthenticationResult(int statusCode) { + this.statusCode = statusCode; + } + + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index bb84c9e17d4..7c06ef27ce9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -16,19 +15,16 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.logging.Logger; -import java.util.stream.Collector; -import java.util.stream.Collectors; +import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toList; /** * This class determines the order of deployments according to an application's deployment spec. @@ -50,6 +46,7 @@ public class DeploymentOrder { /** Returns a list of jobs to trigger after the given job */ // TODO: This does too much - should just tell us the order, as advertised + // TODO: You're next! public List<JobType> nextAfter(JobType job, LockedApplication application) { if ( ! application.deploying().isPresent()) { // Change was cancelled return Collections.emptyList(); @@ -64,7 +61,7 @@ public class DeploymentOrder { // At this point we have deployed to system test, so deployment spec is available List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application); Optional<DeploymentSpec.Step> currentStep = fromJob(job, application); - if (!currentStep.isPresent()) { + if ( ! currentStep.isPresent()) { return Collections.emptyList(); } @@ -75,13 +72,13 @@ public class DeploymentOrder { } // Postpone if step hasn't completed all its jobs for this change - if (!completedSuccessfully(currentStep.get(), application.deploying().get(), application)) { + if ( ! completedSuccessfully(currentStep.get(), application.deploying().get(), application)) { return Collections.emptyList(); } // Postpone next job if delay has not passed yet Duration delay = delayAfter(currentStep.get(), application); - if (postponeDeployment(delay, job, application)) { + if (shouldPostponeDeployment(delay, job, application)) { log.info(String.format("Delaying next job after %s of %s by %s", job, application, delay)); return Collections.emptyList(); } @@ -89,59 +86,40 @@ public class DeploymentOrder { DeploymentSpec.Step nextStep = deploymentSteps.get(currentIndex + 1); return nextStep.zones().stream() .map(this::toJob) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - - /** Returns whether the given job is first in a deployment */ - public boolean isFirst(JobType job) { - return job == JobType.component; - } - - /** Returns whether the given job is last in a deployment */ - public boolean isLast(JobType job, Application application) { - List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application); - if (deploymentSteps.isEmpty()) { // Deployment spec not yet available - return false; - } - DeploymentSpec.Step lastStep = deploymentSteps.get(deploymentSteps.size() - 1); - Optional<DeploymentSpec.Step> step = fromJob(job, application); - // Step may not exist for all jobs, e.g. component - return step.map(s -> s.equals(lastStep)).orElse(false); + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns jobs for given deployment spec, in the order they are declared */ public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) { return deploymentSpec.steps().stream() .flatMap(step -> jobsFrom(step).stream()) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns job status sorted according to deployment spec */ - public Map<JobType, JobStatus> sortBy(DeploymentSpec deploymentSpec, Map<JobType, JobStatus> jobStatus) { - List<DeploymentJobs.JobType> jobs = jobsFrom(deploymentSpec); - return jobStatus.entrySet().stream() - .sorted(Comparator.comparingInt(kv -> jobs.indexOf(kv.getKey()))) - .collect(Collectors.collectingAndThen(toLinkedMap(Map.Entry::getKey, Map.Entry::getValue), - Collections::unmodifiableMap)); + public List<JobStatus> sortBy(DeploymentSpec deploymentSpec, Collection<JobStatus> jobStatus) { + List<DeploymentJobs.JobType> sortedJobs = jobsFrom(deploymentSpec); + return jobStatus.stream() + .sorted(comparingInt(job -> sortedJobs.indexOf(job.type()))) + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns deployments sorted according to declared zones */ - public Map<Zone, Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Map<Zone, Deployment> deployments) { + public List<Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Collection<Deployment> deployments) { List<Zone> productionZones = zones.stream() - .filter(z -> z.environment() == Environment.prod && z.region().isPresent()) + .filter(z -> z.region().isPresent()) .map(z -> new Zone(z.environment(), z.region().get())) - .collect(Collectors.toList()); - return deployments.entrySet().stream() - .sorted(Comparator.comparingInt(kv -> productionZones.indexOf(kv.getKey()))) - .collect(Collectors.collectingAndThen(toLinkedMap(Map.Entry::getKey, Map.Entry::getValue), - Collections::unmodifiableMap)); + .collect(toList()); + return deployments.stream() + .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns jobs for the given step */ private List<JobType> jobsFrom(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); } /** Returns whether all jobs have completed successfully for given step */ @@ -162,11 +140,12 @@ public class DeploymentOrder { /** Resolve job from deployment step */ private JobType toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)); + return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null)) + .orElseThrow(() -> new IllegalArgumentException("Invalid zone " + zone)); } /** Returns whether deployment should be postponed according to delay */ - private boolean postponeDeployment(Duration delay, JobType job, Application application) { + private boolean shouldPostponeDeployment(Duration delay, JobType job, Application application) { Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(job)) .flatMap(JobStatus::lastSuccess) .map(JobStatus.JobRun::at); @@ -176,9 +155,8 @@ public class DeploymentOrder { /** Find all steps that deploy to one or more zones */ private static List<DeploymentSpec.Step> deploymentSteps(Application application) { return application.deploymentSpec().steps().stream() - .filter(step -> step instanceof DeploymentSpec.DeclaredZone || - step instanceof DeploymentSpec.ParallelZones) - .collect(Collectors.toList()); + .filter(step -> ! step.zones().isEmpty()) + .collect(toList()); } /** Determines the delay that should pass after the given step */ @@ -191,7 +169,7 @@ public class DeploymentOrder { List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps() .subList(stepIndex + 1, application.deploymentSpec().steps().size()); for (DeploymentSpec.Step s : remainingSteps) { - if (!(s instanceof DeploymentSpec.Delay)) { + if (! (s instanceof DeploymentSpec.Delay)) { break; } totalDelay = totalDelay.plus(((DeploymentSpec.Delay) s).duration()); @@ -199,13 +177,4 @@ public class DeploymentOrder { return totalDelay; } - private static <T, K, U> Collector<T, ?, Map<K,U>> toLinkedMap(Function<? super T, ? extends K> keyMapper, - Function<? super T, ? extends U> valueMapper) { - return Collectors.toMap(keyMapper, valueMapper, - (u, v) -> { - throw new IllegalStateException(String.format("Duplicate key %s", u)); - }, - LinkedHashMap::new); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e5c7b7f1c2f..1eee727214b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Lock; @@ -14,11 +12,12 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Change.VersionChange; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -26,10 +25,10 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; import java.util.logging.Logger; /** @@ -66,7 +65,11 @@ public class DeploymentTrigger { /** Returns the time in the past before which jobs are at this moment considered unresponsive */ public Instant jobTimeoutLimit() { return clock.instant().minus(jobTimeout); } - + + public BuildSystem buildSystem() { return buildSystem; } + + public DeploymentOrder deploymentOrder() { return order; } + //--- Start of methods which triggers deployment jobs ------------------------- /** @@ -76,16 +79,15 @@ public class DeploymentTrigger { * @param report information about the job that just completed */ public void triggerFromCompletion(JobReport report) { - try (Lock lock = applications().lock(report.applicationId())) { - LockedApplication application = applications().require(report.applicationId(), lock); + applications().lockedOrThrow(report.applicationId(), application -> { application = application.withJobCompletion(report, clock.instant(), controller); - + // Handle successful starting and ending if (report.success()) { - if (order.isFirst(report.jobType())) { // the first job tells us that a change occurred + if (report.jobType() == JobType.component) { if (acceptNewRevisionNow(application)) { // Set this as the change we are doing, unless we are already pushing a platform change - if ( ! ( application.deploying().isPresent() && + if ( ! ( application.deploying().isPresent() && (application.deploying().get() instanceof Change.VersionChange))) application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown())); } @@ -93,8 +95,8 @@ public class DeploymentTrigger { applications().store(application.withOutstandingChange(true)); return; } - } - else if (order.isLast(report.jobType(), application) && application.deployingCompleted()) { + } + else if (deploymentComplete(application)) { // change completed application = application.withDeploying(Optional.empty()); } @@ -103,19 +105,24 @@ public class DeploymentTrigger { // Trigger next if (report.success()) application = trigger(order.nextAfter(report.jobType(), application), application, - String.format("%s completed successfully in build %d", - report.jobType(), report.buildNumber())); - else if (isCapacityConstrained(report.jobType()) && shouldRetryOnOutOfCapacity(application, report.jobType())) + report.jobType().jobName() + " completed"); + else if (retryBecauseOutOfCapacity(application, report.jobType())) application = trigger(report.jobType(), application, true, - String.format("Retrying due to out of capacity in build %d", - report.buildNumber())); - else if (shouldRetryNow(application)) + "Retrying on out of capacity"); + else if (retryBecauseNewFailure(application, report.jobType())) application = trigger(report.jobType(), application, false, - String.format("Retrying as build %d just started failing", - report.buildNumber())); + "Immediate retry on failure"); applications().store(application); - } + }); + } + + /** Returns whether all production zones listed in deployment spec last were successful on the currently deploying change. */ + private boolean deploymentComplete(LockedApplication application) { + if ( ! application.deploying().isPresent()) return true; + return order.jobsFrom(application.deploymentSpec()).stream() + .filter(JobType::isProduction) + .allMatch(jobType -> application.deploymentJobs().isSuccessful(application.deploying().get(), jobType)); } /** @@ -124,13 +131,8 @@ public class DeploymentTrigger { public void triggerReadyJobs() { ApplicationList applications = ApplicationList.from(applications().asList()); applications = applications.notPullRequest(); - for (Application application : applications.asList()) { - try (Lock lock = applications().lock(application.id())) { - Optional<LockedApplication> lockedApplication = controller.applications().get(application.id(), lock); - if ( ! lockedApplication.isPresent()) continue; // application removed - triggerReadyJobs(lockedApplication.get()); - } - } + for (Application application : applications.asList()) + applications().lockedIfPresent(application.id(), this::triggerReadyJobs); } /** Find the next step to trigger if any, and triggers it */ @@ -139,14 +141,24 @@ public class DeploymentTrigger { List<JobType> jobs = order.jobsFrom(application.deploymentSpec()); // Should the first step be triggered? - if ( ! jobs.isEmpty() && jobs.get(0).equals(JobType.systemTest) && - application.deploying().get() instanceof Change.VersionChange) { - Version target = ((Change.VersionChange)application.deploying().get()).version(); - JobStatus jobStatus = application.deploymentJobs().jobStatus().get(JobType.systemTest); - if (jobStatus == null || ! jobStatus.lastTriggered().isPresent() - || ! jobStatus.lastTriggered().get().version().equals(target)) { - application = trigger(JobType.systemTest, application, false, "Upgrade to " + target); - controller.applications().store(application); + if ( ! jobs.isEmpty() && jobs.get(0).equals(JobType.systemTest) ) { + JobStatus systemTestStatus = application.deploymentJobs().jobStatus().get(JobType.systemTest); + if (application.deploying().get() instanceof Change.VersionChange) { + Version target = ((Change.VersionChange) application.deploying().get()).version(); + if (systemTestStatus == null + || ! systemTestStatus.lastTriggered().isPresent() + || ! systemTestStatus.isSuccess() + || ! systemTestStatus.lastTriggered().get().version().equals(target)) { + application = trigger(JobType.systemTest, application, false, "Upgrade to " + target); + controller.applications().store(application); + } + } + else { + JobStatus componentStatus = application.deploymentJobs().jobStatus().get(JobType.component); + if (changesAvailable(application, componentStatus, systemTestStatus)) { + application = trigger(JobType.systemTest, application, false, "Available change in component"); + controller.applications().store(application); + } } } @@ -164,7 +176,7 @@ public class DeploymentTrigger { nextToTrigger.add(nextJobType); } // Trigger them in parallel - application = trigger(nextToTrigger, application, "Triggering previously blocked jobs"); + application = trigger(nextToTrigger, application, "Available change in " + jobType.jobName()); controller.applications().store(application); } } @@ -177,14 +189,14 @@ public class DeploymentTrigger { if ( ! application.deploying().isPresent()) return false; Change change = application.deploying().get(); - if ( ! previous.lastSuccess().isPresent() && - ! productionJobHasSucceededFor(previous, change)) return false; + if ( ! previous.lastSuccess().isPresent()) return false; if (change instanceof Change.VersionChange) { Version targetVersion = ((Change.VersionChange)change).version(); if ( ! (targetVersion.equals(previous.lastSuccess().get().version())) ) return false; // version is outdated - if (isOnNewerVersionInProductionThan(targetVersion, application, next.type())) + // The below is checked again in allowedTriggering, right before actual triggering. + if (next != null && isOnNewerVersionInProductionThan(targetVersion, application, next.type())) return false; // Don't downgrade } @@ -193,7 +205,7 @@ public class DeploymentTrigger { JobStatus.JobRun previousSuccess = previous.lastSuccess().get(); JobStatus.JobRun nextSuccess = next.lastSuccess().get(); - if (previousSuccess.revision().isPresent() && ! previousSuccess.revision().get().equals(nextSuccess.revision().get())) + if (previousSuccess.revision().isPresent() && ! previousSuccess.revision().equals(nextSuccess.revision())) return true; if ( ! previousSuccess.version().equals(nextSuccess.version())) return true; @@ -201,79 +213,23 @@ public class DeploymentTrigger { } /** - * Called periodically to cause triggering of jobs in the background - */ - public void triggerFailing(ApplicationId applicationId) { - try (Lock lock = applications().lock(applicationId)) { - LockedApplication application = applications().require(applicationId, lock); - if ( ! application.deploying().isPresent()) return; // No ongoing change, no need to retry - - // Retry first failing job - for (JobType jobType : order.jobsFrom(application.deploymentSpec())) { - JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); - if (isFailing(application.deploying().get(), jobStatus)) { - if (shouldRetryNow(jobStatus)) { - application = trigger(jobType, application, false, "Retrying failing job"); - applications().store(application); - } - break; - } - } - - // Retry dead job - Optional<JobStatus> firstDeadJob = firstDeadJob(application.deploymentJobs()); - if (firstDeadJob.isPresent()) { - application = trigger(firstDeadJob.get().type(), application, false, "Retrying dead job"); - applications().store(application); - } - } - } - - /** Triggers jobs that have been delayed according to deployment spec */ - public void triggerDelayed() { - for (Application application : applications().asList()) { - if ( ! application.deploying().isPresent() ) continue; - if (application.deploymentJobs().hasFailures()) continue; - if (application.deploymentJobs().isRunning(controller.applications().deploymentTrigger().jobTimeoutLimit())) continue; - if (application.deploymentSpec().steps().stream().noneMatch(step -> step instanceof DeploymentSpec.Delay)) { - continue; // Application does not have any delayed deployments - } - - Optional<JobStatus> lastSuccessfulJob = application.deploymentJobs().jobStatus().values() - .stream() - .filter(j -> j.lastSuccess().isPresent()) - .sorted(Comparator.<JobStatus, Instant>comparing(j -> j.lastSuccess().get().at()).reversed()) - .findFirst(); - if ( ! lastSuccessfulJob.isPresent() ) continue; - - // Trigger next - try (Lock lock = applications().lock(application.id())) { - LockedApplication lockedApplication = applications().require(application.id(), lock); - lockedApplication = trigger(order.nextAfter(lastSuccessfulJob.get().type(), lockedApplication), - lockedApplication, "Resuming delayed deployment"); - applications().store(lockedApplication); - } - } - } - - /** * Triggers a change of this application * * @param applicationId the application to trigger * @throws IllegalArgumentException if this application already have an ongoing change */ public void triggerChange(ApplicationId applicationId, Change change) { - try (Lock lock = applications().lock(applicationId)) { - LockedApplication application = applications().require(applicationId, lock); + applications().lockedOrThrow(applicationId, application -> { if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) - throw new IllegalArgumentException("Could not start " + change + " on " + application + ": " + + throw new IllegalArgumentException("Could not start " + change + " on " + application + ": " + application.deploying().get() + " is already in progress"); application = application.withDeploying(Optional.of(change)); if (change instanceof Change.ApplicationChange) application = application.withOutstandingChange(false); - application = trigger(JobType.systemTest, application, false, "Deploying change"); + application = trigger(JobType.systemTest, application, false, + (change instanceof Change.VersionChange ? "Upgrading to " + ((Change.VersionChange)change).version() : "Deploying " + change)); applications().store(application); - } + }); } /** @@ -282,81 +238,34 @@ public class DeploymentTrigger { * @param applicationId the application to trigger */ public void cancelChange(ApplicationId applicationId) { - try (Lock lock = applications().lock(applicationId)) { - LockedApplication application = applications().require(applicationId, lock); + applications().lockedOrThrow(applicationId, application -> { buildSystem.removeJobs(application.id()); - application = application.withDeploying(Optional.empty()); - applications().store(application); - } + applications().store(application.withDeploying(Optional.empty())); + }); } //--- End of methods which triggers deployment jobs ---------------------------- private ApplicationController applications() { return controller.applications(); } - /** Returns whether a job is failing for the current change in the given application */ - private boolean isFailing(Change change, JobStatus status) { - return status != null && - !status.isSuccess() && - status.lastCompletedFor(change); - } - - private boolean isCapacityConstrained(JobType jobType) { - return jobType == JobType.stagingTest || jobType == JobType.systemTest; - } - - /** Returns the first job that has been running for more than the given timeout */ - private Optional<JobStatus> firstDeadJob(DeploymentJobs jobs) { - Optional<JobStatus> oldestRunningJob = jobs.jobStatus().values().stream() - .filter(job -> job.isRunning(Instant.ofEpochMilli(0))) - .sorted(Comparator.comparing(status -> status.lastTriggered().get().at())) - .findFirst(); - return oldestRunningJob.filter(job -> job.lastTriggered().get().at().isBefore(jobTimeoutLimit())); - } - - /** Decide whether the job should be triggered by the periodic trigger */ - private boolean shouldRetryNow(JobStatus job) { - if (job.isSuccess()) return false; - if (job.isRunning(jobTimeoutLimit())) return false; - - // Retry after 10% of the time since it started failing - Duration aTenthOfFailTime = Duration.ofMillis( (clock.millis() - job.firstFailing().get().at().toEpochMilli()) / 10); - if (job.lastCompleted().get().at().isBefore(clock.instant().minus(aTenthOfFailTime))) return true; - - // ... or retry anyway if we haven't tried in 4 hours - if (job.lastCompleted().get().at().isBefore(clock.instant().minus(Duration.ofHours(4)))) return true; - - return false; - } - - /** Retry immediately only if this just started failing. Otherwise retry periodically */ - private boolean shouldRetryNow(Application application) { - return application.deploymentJobs().failingSince().isAfter(clock.instant().minus(Duration.ofSeconds(10))); + /** Retry immediately only if this job just started failing. Otherwise retry periodically */ + private boolean retryBecauseNewFailure(Application application, JobType jobType) { + JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); + return (jobStatus != null && jobStatus.firstFailing().get().at().isAfter(clock.instant().minus(Duration.ofSeconds(10)))); } /** Decide whether to retry due to capacity restrictions */ - private boolean shouldRetryOnOutOfCapacity(Application application, JobType jobType) { - Optional<JobError> outOfCapacityError = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType)) - .flatMap(JobStatus::jobError) - .filter(e -> e.equals(JobError.outOfCapacity)); - - if ( ! outOfCapacityError.isPresent()) return false; - + private boolean retryBecauseOutOfCapacity(Application application, JobType jobType) { + JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); + if (jobStatus == null || ! jobStatus.jobError().equals(Optional.of(JobError.outOfCapacity))) return false; // Retry the job if it failed recently - return application.deploymentJobs().jobStatus().get(jobType).firstFailing().get().at() - .isAfter(clock.instant().minus(Duration.ofMinutes(15))); + return jobStatus.firstFailing().get().at().isAfter(clock.instant().minus(Duration.ofMinutes(15))); } /** Returns whether the given job type should be triggered according to deployment spec */ - private boolean deploysTo(Application application, JobType jobType) { - Optional<Zone> zone = jobType.zone(controller.system()); - if (zone.isPresent() && jobType.isProduction()) { - // Skip triggering of jobs for zones where the application should not be deployed - if ( ! application.deploymentSpec().includes(jobType.environment(), Optional.of(zone.get().region()))) { - return false; - } - } - return true; + private boolean hasJob(JobType jobType, Application application) { + if ( ! jobType.isProduction()) return true; // Deployment spec only determines this for production jobs. + return application.deploymentSpec().includes(jobType.environment(), jobType.region(controller.system())); } /** @@ -364,19 +273,19 @@ public class DeploymentTrigger { * * @param jobType the type of the job to trigger, or null to trigger nothing * @param application the application to trigger the job for - * @param first whether to trigger the job before other jobs - * @param cause describes why the job is triggered + * @param first whether to put the job at the front of the build system queue (or the back) + * @param reason describes why the job is triggered * @return the application in the triggered state, which *must* be stored by the caller */ - private LockedApplication trigger(JobType jobType, LockedApplication application, boolean first, String cause) { - if (isRunningProductionJob(application)) return application; - return triggerAllowParallel(jobType, application, first, false, cause); + private LockedApplication trigger(JobType jobType, LockedApplication application, boolean first, String reason) { + if (jobType.isProduction() && isRunningProductionJob(application)) return application; + return triggerAllowParallel(jobType, application, first, false, reason); } - private LockedApplication trigger(List<JobType> jobs, LockedApplication application, String cause) { - if (isRunningProductionJob(application)) return application; + private LockedApplication trigger(List<JobType> jobs, LockedApplication application, String reason) { + if (jobs.stream().anyMatch(JobType::isProduction) && isRunningProductionJob(application)) return application; for (JobType job : jobs) - application = triggerAllowParallel(job, application, false, false, cause); + application = triggerAllowParallel(job, application, false, false, reason); return application; } @@ -406,8 +315,12 @@ public class DeploymentTrigger { application.deploying().map(d -> "deploying " + d).orElse("restarted deployment"), reason)); buildSystem.addJob(application.id(), jobType, first); - return application.withJobTriggering(-1, jobType, application.deploying(), reason, clock.instant(), - controller); + return application.withJobTriggering(jobType, + application.deploying(), + clock.instant(), + application.deployVersionFor(jobType, controller), + application.deployRevisionFor(jobType, controller), + reason); } /** Returns true if the given proposed job triggering should be effected */ @@ -416,37 +329,29 @@ public class DeploymentTrigger { // by instead basing the decision on what is currently deployed in the zone. However, // this leads to some additional corner cases, and the possibility of blocking an application // fix to a version upgrade, so not doing it now - if (jobType.isProduction() && application.deployingBlocked(clock.instant())) return false; + + if (jobType.isProduction() && application.deploying().isPresent() && + application.deploying().get().blockedBy(application.deploymentSpec(), clock.instant())) return false; + + if (application.deploying().isPresent() && application.deploying().get() instanceof VersionChange && + isOnNewerVersionInProductionThan(((VersionChange) application.deploying().get()).version(), application, jobType)) return false; + if (application.deploymentJobs().isRunning(jobType, jobTimeoutLimit())) return false; - if ( ! deploysTo(application, jobType)) return false; + if ( ! hasJob(jobType, application)) return false; // Ignore applications that are not associated with a project if ( ! application.deploymentJobs().projectId().isPresent()) return false; - if (application.deploying().isPresent() && application.deploying().get() instanceof Change.VersionChange) { - Version targetVersion = ((Change.VersionChange)application.deploying().get()).version(); - if (isOnNewerVersionInProductionThan(targetVersion, application, jobType)) return false; // Don't downgrade - } - + return true; } private boolean isRunningProductionJob(Application application) { - return application.deploymentJobs().jobStatus().entrySet().stream() - .anyMatch(entry -> entry.getKey().isProduction() && entry.getValue().isRunning(jobTimeoutLimit())); + return JobList.from(application) + .production() + .running(jobTimeoutLimit()) + .anyMatch(); } /** - * When upgrading it is ok to trigger the next job even if the previous failed if the previous has earlier succeeded - * on the version we are currently upgrading to - */ - private boolean productionJobHasSucceededFor(JobStatus jobStatus, Change change) { - if ( ! (change instanceof Change.VersionChange) ) return false; - if ( ! isProduction(jobStatus.type())) return false; - Optional<JobStatus.JobRun> lastSuccess = jobStatus.lastSuccess(); - if ( ! lastSuccess.isPresent()) return false; - return lastSuccess.get().version().equals(((Change.VersionChange)change).version()); - } - - /** * Returns whether the current deployed version in the zone given by the job * is newer than the given version. This may be the case even if the production job * in question failed, if the failure happens after deployment. @@ -454,7 +359,7 @@ public class DeploymentTrigger { * downgrade production nodes which we are not guaranteed to support. */ private boolean isOnNewerVersionInProductionThan(Version version, Application application, JobType job) { - if ( ! isProduction(job)) return false; + if ( ! job.isProduction()) return false; Optional<Zone> zone = job.zone(controller.system()); if ( ! zone.isPresent()) return false; Deployment existingDeployment = application.deployments().get(zone.get()); @@ -462,23 +367,17 @@ public class DeploymentTrigger { return existingDeployment.version().isAfter(version); } - private boolean isProduction(JobType job) { - Optional<Zone> zone = job.zone(controller.system()); - if ( ! zone.isPresent()) return false; // arbitrary - return zone.get().environment() == Environment.prod; - } - private boolean acceptNewRevisionNow(LockedApplication application) { if ( ! application.deploying().isPresent()) return true; - if ( application.deploying().get() instanceof Change.ApplicationChange) return true; // more changes are ok - - if ( application.deploymentJobs().hasFailures()) return true; // allow changes to fix upgrade problems - if ( application.isBlocked(clock.instant())) return true; // allow testing changes while upgrade blocked (debatable) + + if (application.deploying().get() instanceof Change.ApplicationChange) return true; // more changes are ok + + if (application.deploymentJobs().hasFailures()) return true; // allow changes to fix upgrade problems + + if (application.isBlocked(clock.instant())) return true; // allow testing changes while upgrade blocked (debatable) + + // Otherwise, the application is currently upgrading, without failures, and we should wait with the revision. return false; } - public BuildSystem buildSystem() { return buildSystem; } - - public DeploymentOrder deploymentOrder() { return order; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java index 56b4023f932..e25db10a8cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java @@ -87,9 +87,9 @@ public class PolledBuildSystem implements BuildSystem { Optional<Long> projectId = projectId(application); if (projectId.isPresent()) { - jobsToRun.add(new BuildJob(projectId.get(), jobType.id())); + jobsToRun.add(new BuildJob(projectId.get(), jobType.jobName())); } else { - log.warning("Not queuing " + jobType.id() + " for " + application.toShortString() + + log.warning("Not queuing " + jobType.jobName() + " for " + application.toShortString() + " because project ID is missing"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java new file mode 100644 index 00000000000..09f8df58205 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -0,0 +1,92 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; + +import java.time.Duration; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.logging.Level; + +/** + * Periodically request application ownership confirmation through filing issues. + * + * When to file new issues, escalate inactive ones, etc., is handled by the enclosed OwnershipIssues. + * + * @author jvenstad + */ +public class ApplicationOwnershipConfirmer extends Maintainer { + + private final OwnershipIssues ownershipIssues; + + public ApplicationOwnershipConfirmer(Controller controller, Duration interval, JobControl jobControl, OwnershipIssues ownershipIssues) { + super(controller, interval, jobControl); + this.ownershipIssues = ownershipIssues; + } + + @Override + protected void maintain() { + confirmApplicationOwnerships(); + ensureConfirmationResponses(); + } + + /** File an ownership issue with the owners of all applications we know about. */ + private void confirmApplicationOwnerships() { + for (Application application : controller().applications().asList()) + if (application.id().instance().value().startsWith("default-pr") || application.productionDeployments().isEmpty()) + store(null, application.id()); + else + try { + Tenant tenant = ownerOf(application.id()); + Optional<IssueId> ourIssueId = application.ownershipIssueId(); + ourIssueId = tenant.tenantType() == TenantType.USER + ? ownershipIssues.confirmOwnership(ourIssueId, application.id(), userFor(tenant)) + : ownershipIssues.confirmOwnership(ourIssueId, application.id(), propertyIdFor(tenant)); + ourIssueId.ifPresent(issueId -> store(issueId, application.id())); + } + catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. + log.log(Level.WARNING, "Exception caught when attempting to file an issue for " + application.id(), e); + } + } + + /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ + private void ensureConfirmationResponses() { + for (Application application : controller().applications().asList()) + application.ownershipIssueId().ifPresent(issueId -> { + try { + ownershipIssues.ensureResponse(issueId, ownerOf(application.id()).getPropertyId()); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Exception caught when attempting to escalate issue with id " + issueId, e); + } + }); + } + + private Tenant ownerOf(ApplicationId applicationId) { + return controller().tenants().tenant(new TenantId(applicationId.tenant().value())) + .orElseThrow(() -> new IllegalStateException("No tenant found for application " + applicationId)); + } + + protected User userFor(Tenant tenant) { + return User.from(tenant.getId().id().replaceFirst("by-", "")); + } + + protected PropertyId propertyIdFor(Tenant tenant) { + return tenant.getPropertyId() + .orElseThrow(() -> new NoSuchElementException("No PropertyId is listed for non-user tenant " + tenant)); + } + + protected void store(IssueId issueId, ApplicationId applicationId) { + controller().applications().lockedIfPresent(applicationId, application -> + controller().applications().store(application.withOwnershipIssueId(issueId))); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index 275aedfc812..ae617f87be6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -86,22 +87,17 @@ public class ClusterInfoMaintainer extends Maintainer { @Override protected void maintain() { - for (Application application : controller().applications().asList()) { - try (Lock lock = controller().applications().lock(application.id())) { - Optional<LockedApplication> lockedApplication = controller.applications().get(application.id(), lock); - if (!lockedApplication.isPresent()) continue; // application removed - - for (Deployment deployment : lockedApplication.get().deployments().values()) { - DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); - try { - NodeList nodes = controller().applications().configserverClient().getNodeList(deploymentId); - Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes, deployment.zone()); - controller.applications().store(lockedApplication.get() - .with(deployment.withClusterInfo(clusterInfo))); - } - catch (IOException | IllegalArgumentException e) { - log.log(Level.WARNING, "Failing getting cluster info of for " + deploymentId, e); - } + for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { + for (Deployment deployment : application.deployments().values()) { + DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); + try { + NodeList nodes = controller().applications().configserverClient().getNodeList(deploymentId); + Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes, deployment.zone()); + controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo))); + } + catch (IOException | IllegalArgumentException e) { + log.log(Level.WARNING, "Failing getting cluster info of for " + deploymentId, e); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 60b890f10fb..3744be67135 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -7,15 +7,14 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import java.time.Duration; import java.util.HashMap; import java.util.Map; -import java.util.Optional; /** * Fetch utilization metrics and update applications with this data. @@ -46,16 +45,15 @@ public class ClusterUtilizationMaintainer extends Maintainer { @Override protected void maintain() { - for (Application application : controller().applications().asList()) { - try (Lock lock = controller().applications().lock(application.id())) { - Optional<LockedApplication> lockedApplication = controller.applications().get(application.id(), lock); - if (!lockedApplication.isPresent()) continue; // application removed - for (Deployment deployment : application.deployments().values()) { - Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); - controller.applications().store(lockedApplication.get() - .with(deployment.withClusterUtils(clusterUtilization))); - } + for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { + for (Deployment deployment : application.deployments().values()) { + + Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); + + controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().store(lockedApplication.withClusterUtilization(deployment.zone(), clusterUtilization))); } } } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 2fdce2802ab..bc2112ac0ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; @@ -25,34 +26,32 @@ public class ControllerMaintenance extends AbstractComponent { private final DeploymentExpirer deploymentExpirer; private final DeploymentIssueReporter deploymentIssueReporter; private final MetricsReporter metricsReporter; - private final FailureRedeployer failureRedeployer; private final OutstandingChangeDeployer outstandingChangeDeployer; private final VersionStatusUpdater versionStatusUpdater; private final Upgrader upgrader; - private final DelayedDeployer delayedDeployer; - private final BlockedChangeDeployer blockedChangeDeployer; + private final ReadyJobsTrigger readyJobsTrigger; private final ClusterInfoMaintainer clusterInfoMaintainer; private final ClusterUtilizationMaintainer clusterUtilizationMaintainer; private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; + private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, JobControl jobControl, Metric metric, Chef chefClient, - DeploymentIssues deploymentIssues) { + DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); deploymentIssueReporter = new DeploymentIssueReporter(controller, deploymentIssues, maintenanceInterval, jobControl); metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); - failureRedeployer = new FailureRedeployer(controller, maintenanceInterval, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl); versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); - delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl); - blockedChangeDeployer = new BlockedChangeDeployer(controller, maintenanceInterval, jobControl); + readyJobsTrigger = new ReadyJobsTrigger(controller, maintenanceInterval, jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl); clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); + applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); } public Upgrader upgrader() { return upgrader; } @@ -65,15 +64,14 @@ public class ControllerMaintenance extends AbstractComponent { deploymentExpirer.deconstruct(); deploymentIssueReporter.deconstruct(); metricsReporter.deconstruct(); - failureRedeployer.deconstruct(); outstandingChangeDeployer.deconstruct(); versionStatusUpdater.deconstruct(); upgrader.deconstruct(); - delayedDeployer.deconstruct(); - blockedChangeDeployer.deconstruct(); + readyJobsTrigger.deconstruct(); clusterUtilizationMaintainer.deconstruct(); clusterInfoMaintainer.deconstruct(); deploymentMetricsMaintainer.deconstruct(); + applicationOwnershipConfirmer.deconstruct(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java deleted file mode 100644 index cb09c41a034..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Duration; - -/** - * Maintenance job which triggers jobs that have been delayed according to the applications deployment spec. - * - * @author mpolden - */ -public class DelayedDeployer extends Maintainer { - - public DelayedDeployer(Controller controller, Duration interval, JobControl jobControl) { - super(controller, interval, jobControl); - } - - @Override - protected void maintain() { - controller().applications().deploymentTrigger().triggerDelayed(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index b4708dccb6b..ae6ba364d25 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -67,7 +67,7 @@ public class DeploymentIssueReporter extends Maintainer { if (failingApplications.contains(application.id())) fileDeploymentIssueFor(application.id()); else - storeIssueId(application.id(), null); + store(application.id(), null); } /** @@ -111,7 +111,7 @@ public class DeploymentIssueReporter extends Maintainer { IssueId issueId = tenant.tenantType() == TenantType.USER ? deploymentIssues.fileUnlessOpen(ourIssueId, applicationId, userFor(tenant)) : deploymentIssues.fileUnlessOpen(ourIssueId, applicationId, propertyIdFor(tenant)); - storeIssueId(applicationId, issueId); + store(applicationId, issueId); } catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. log.log(Level.WARNING, "Exception caught when attempting to file an issue for " + applicationId, e); @@ -130,12 +130,9 @@ public class DeploymentIssueReporter extends Maintainer { })); } - private void storeIssueId(ApplicationId id, IssueId issueId) { - try (Lock lock = controller().applications().lock(id)) { - controller().applications().get(id, lock).ifPresent( - application -> controller().applications().store(application.with(issueId)) - ); - } + private void store(ApplicationId id, IssueId issueId) { + controller().applications().lockedIfPresent(id, application -> + controller().applications().store(application.withDeploymentIssueId(issueId))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 2e6e378272d..13eb5075f34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -4,15 +4,14 @@ package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Ho import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.yolean.Exceptions; import java.io.UncheckedIOException; import java.time.Duration; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -33,38 +32,31 @@ public class DeploymentMetricsMaintainer extends Maintainer { @Override protected void maintain() { boolean hasWarned = false; - for (Application application : controller().applications().asList()) { - for (Deployment deployment : application.deployments().values()) { - try { - MetricsService.DeploymentMetrics metrics = controller().metricsService() - .getDeploymentMetrics(application.id(), deployment.zone()); - DeploymentMetrics appMetrics = new DeploymentMetrics(metrics.queriesPerSecond(), metrics.writesPerSecond(), - metrics.documentCount(), metrics.queryLatencyMillis(), metrics.writeLatencyMillis()); - - try (Lock lock = controller().applications().lock(application.id())) { - - // Deployment or application may have changed (or be gone) now: - Optional<LockedApplication> lockedApplication = controller().applications() - .get(application.id(), lock); - if (!lockedApplication.isPresent()) continue; + for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { + try { + controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().store(lockedApplication.with(controller().metricsService().getApplicationMetrics(application.id())))); - deployment = lockedApplication.get().deployments().get(deployment.zone()); - if (deployment == null) continue; - - controller().applications().store(lockedApplication.get() - .with(deployment.withMetrics(appMetrics))); - } - } - catch (UncheckedIOException e) { - if ( ! hasWarned) // produce only one warning per maintenance interval - log.log(Level.WARNING, "Failed talking to YAMAS: " + Exceptions.toMessageString(e) + - ". Retrying in " + maintenanceInterval()); - hasWarned = true; + for (Deployment deployment : application.deployments().values()) { + MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() + .getDeploymentMetrics(application.id(), deployment.zone()); + DeploymentMetrics appMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(), + deploymentMetrics.writesPerSecond(), + deploymentMetrics.documentCount(), + deploymentMetrics.queryLatencyMillis(), + deploymentMetrics.writeLatencyMillis()); + + controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().store(lockedApplication.with(deployment.zone(), appMetrics))); } } + catch (UncheckedIOException e) { + if (!hasWarned) // produce only one warning per maintenance interval + log.log(Level.WARNING, "Failed talking to YAMAS: " + Exceptions.toMessageString(e) + + ". Retrying in " + maintenanceInterval()); + hasWarned = true; + } } - } } - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java deleted file mode 100644 index 72f8faa5180..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; - -import java.time.Duration; -import java.util.List; - -/** - * Attempts redeployment of failed jobs and deployments. - * - * @author bratseth - * @author mpolden - */ -public class FailureRedeployer extends Maintainer { - - public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) { - super(controller, interval, jobControl); - } - - @Override - public void maintain() { - List<Application> applications = ApplicationList.from(controller().applications().asList()) - .notPullRequest() - .asList(); - applications.forEach(application -> triggerFailing(application)); - } - - private void triggerFailing(Application application) { - controller().applications().deploymentTrigger().triggerFailing(application.id()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java index d7396cb2acb..6aa1b89c605 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; import java.util.logging.Logger; @@ -40,7 +41,7 @@ public class JobControl { * Returns a snapshot of the set of jobs started on this system (whether deactivated or not). * Each job is represented by its simple (omitting package) class name. */ - public Set<String> jobs() { return new HashSet<>(startedJobs); } + public Set<String> jobs() { return new LinkedHashSet<>(startedJobs); } /** Returns an unmodifiable set containing the currently inactive jobs in this */ public Set<String> inactiveJobs() { return curator.readInactiveJobs(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java index bbef7980273..ebab2054d4f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 3d0cd284c55..01e53ce4f79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; import java.time.Clock; import java.time.Duration; @@ -102,9 +103,12 @@ public class MetricsReporter extends Maintainer { } private double deploymentFailRatio() { - List<Application> applications = controller().applications().asList(); + List<Application> applications = ApplicationList.from(controller().applications().asList()) + .notPullRequest() + .hasProductionDeployment() + .asList(); if (applications.isEmpty()) return 0; - + return (double)applications.stream().filter(a -> a.deploymentJobs().hasFailures()).count() / (double)applications.size(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BlockedChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index 4a68fd6cfab..f165b4e4ea3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BlockedChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -14,14 +14,14 @@ import java.time.Duration; * @author bratseth */ @SuppressWarnings("unused") -public class BlockedChangeDeployer extends Maintainer { +public class ReadyJobsTrigger extends Maintainer { - public BlockedChangeDeployer(Controller controller, Duration interval, JobControl jobControl) { + public ReadyJobsTrigger(Controller controller, Duration interval, JobControl jobControl) { super(controller, interval, jobControl); } @Override - protected void maintain() { + public void maintain() { controller().applications().deploymentTrigger().triggerReadyJobs(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 36b87e4cead..5b87f9eaa86 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -26,8 +26,6 @@ import java.util.logging.Logger; */ public class Upgrader extends Maintainer { - private static final Duration upgradeTimeout = Duration.ofHours(12); - private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; @@ -41,23 +39,32 @@ public class Upgrader extends Maintainer { * Schedule application upgrades. Note that this implementation must be idempotent. */ @Override - public void maintain() { - ApplicationList applications = applications(); - + public void maintain() { // Determine target versions for each upgrade policy Optional<Version> canaryTarget = controller().versionStatus().systemVersion().map(VespaVersion::versionNumber); Optional<Version> defaultTarget = newestVersionWithConfidence(VespaVersion.Confidence.normal); Optional<Version> conservativeTarget = newestVersionWithConfidence(VespaVersion.Confidence.high); - // Cancel any upgrades to the wrong targets - cancelUpgradesOf(applications.with(UpgradePolicy.canary).upgrading().notUpgradingTo(canaryTarget)); - cancelUpgradesOf(applications.with(UpgradePolicy.defaultPolicy).upgrading().notUpgradingTo(defaultTarget)); - cancelUpgradesOf(applications.with(UpgradePolicy.conservative).upgrading().notUpgradingTo(conservativeTarget)); + // Cancel upgrades to broken targets (let other ongoing upgrades complete to avoid starvation + for (VespaVersion version : controller().versionStatus().versions()) { + if (version.confidence() == VespaVersion.Confidence.broken) + cancelUpgradesOf(applications().without(UpgradePolicy.canary).upgradingTo(version.versionNumber()), + version.versionNumber() + " is broken"); + } + + // Canaries should always try the canary target + cancelUpgradesOf(applications().with(UpgradePolicy.canary).upgrading().notUpgradingTo(canaryTarget), + "Outdated target version for Canaries"); + + // Cancel *failed* upgrades to earlier versions, as the new version may fix it + String reason = "Failing on outdated version"; + cancelUpgradesOf(applications().with(UpgradePolicy.defaultPolicy).upgrading().failing().notUpgradingTo(defaultTarget), reason); + cancelUpgradesOf(applications().with(UpgradePolicy.conservative).upgrading().failing().notUpgradingTo(conservativeTarget), reason); // Schedule the right upgrades - canaryTarget.ifPresent(target -> upgrade(applications.with(UpgradePolicy.canary), target)); - defaultTarget.ifPresent(target -> upgrade(applications.with(UpgradePolicy.defaultPolicy), target)); - conservativeTarget.ifPresent(target -> upgrade(applications.with(UpgradePolicy.conservative), target)); + canaryTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.canary), target)); + defaultTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.defaultPolicy), target)); + conservativeTarget.ifPresent(target -> upgrade(applications().with(UpgradePolicy.conservative), target)); } private Optional<Version> newestVersionWithConfidence(VespaVersion.Confidence confidence) { @@ -79,13 +86,11 @@ public class Upgrader extends Maintainer { private void upgrade(ApplicationList applications, Version version) { Change.VersionChange change = new Change.VersionChange(version); - cancelUpgradesOf(applications.upgradingToLowerThan(version)); applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade applications = applications.hasProductionDeployment(); applications = applications.onLowerVersionThan(version); - applications = applications.notDeployingApplication(); // wait with applications deploying an application change + applications = applications.notDeploying(); // wait with applications deploying an application change or already upgrading applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version - applications = applications.notCurrentlyUpgrading(change, controller().applications().deploymentTrigger().jobTimeoutLimit()); applications = applications.canUpgradeAt(controller().clock().instant()); // wait with applications that are currently blocking upgrades applications = applications.byIncreasingDeployedVersion(); // start with lowest versions applications = applications.first(numberOfApplicationsToUpgrade()); // throttle upgrades @@ -98,9 +103,9 @@ public class Upgrader extends Maintainer { } } - private void cancelUpgradesOf(ApplicationList applications) { + private void cancelUpgradesOf(ApplicationList applications, String reason) { if (applications.isEmpty()) return; - log.info("Cancelling upgrading of " + applications.asList().size() + " applications"); + log.info("Cancelling upgrading of " + applications.asList().size() + " applications: " + reason); for (Application application : applications.asList()) controller().applications().deploymentTrigger().cancelChange(application.id()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 3bd1abdf607..23316a74aae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -15,6 +15,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; @@ -51,7 +52,10 @@ public class ApplicationSerializer { private final String deploymentJobsField = "deploymentJobs"; private final String deployingField = "deployingField"; private final String outstandingChangeField = "outstandingChangeField"; - + private final String ownershipIssueIdField = "ownershipIssueId"; + private final String writeQualityField = "writeQuality"; + private final String queryQualityField = "queryQuality"; + // Deployment fields private final String zoneField = "zone"; private final String environmentField = "environment"; @@ -123,6 +127,9 @@ public class ApplicationSerializer { toSlime(application.deploymentJobs(), root.setObject(deploymentJobsField)); toSlime(application.deploying(), root); root.setBool(outstandingChangeField, application.hasOutstandingChange()); + application.ownershipIssueId().ifPresent(issueId -> root.setString(ownershipIssueIdField, issueId.value())); + root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); + root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); return slime; } @@ -202,9 +209,7 @@ public class ApplicationSerializer { } private void toSlime(DeploymentJobs deploymentJobs, Cursor cursor) { - deploymentJobs.projectId() - .filter(id -> id > 0) // TODO: Discards invalid data. Remove filter after October 2017 - .ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); + deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); deploymentJobs.issueId().ifPresent(jiraIssueId -> cursor.setString(issueIdField, jiraIssueId.value())); } @@ -215,7 +220,7 @@ public class ApplicationSerializer { } private void toSlime(JobStatus jobStatus, Cursor object) { - object.setString(jobTypeField, jobStatus.type().id()); + object.setString(jobTypeField, jobStatus.type().jobName()); if (jobStatus.jobError().isPresent()) object.setString(errorField, jobStatus.jobError().get().name()); @@ -259,9 +264,12 @@ public class ApplicationSerializer { DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField)); Optional<Change> deploying = changeFromSlime(root.field(deployingField)); boolean outstandingChange = root.field(outstandingChangeField).asBool(); + Optional<IssueId> ownershipIssueId = optionalString(root.field(ownershipIssueIdField)).map(IssueId::from); + ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), + root.field(writeQualityField).asDouble()); - return new Application(id, deploymentSpec, validationOverrides, deployments, - deploymentJobs, deploying, outstandingChange); + return new Application(id, deploymentSpec, validationOverrides, deployments, + deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics); } private List<Deployment> deploymentsFromSlime(Inspector array) { @@ -347,8 +355,7 @@ public class ApplicationSerializer { } private DeploymentJobs deploymentJobsFromSlime(Inspector object) { - Optional<Long> projectId = optionalLong(object.field(projectIdField)) - .filter(id -> id > 0); // TODO: Discards invalid data. Remove filter after October 2017 + Optional<Long> projectId = optionalLong(object.field(projectIdField)); List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField)); Optional<IssueId> issueId = optionalString(object.field(issueIdField)).map(IssueId::from); @@ -373,7 +380,7 @@ public class ApplicationSerializer { } private JobStatus jobStatusFromSlime(Inspector object) { - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromId(object.field(jobTypeField).asString()); + DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromJobName(object.field(jobTypeField).asString()); Optional<JobError> jobError = Optional.empty(); if (object.field(errorField).valid()) @@ -388,7 +395,7 @@ public class ApplicationSerializer { private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); - return Optional.of(new JobStatus.JobRun(optionalLong(object.field(jobRunIdField)).orElse(-1L), // TODO: Make non-optional after November 2017 + return Optional.of(new JobStatus.JobRun(optionalLong(object.field(jobRunIdField)).orElse(-1L), // TODO: Make non-optional after November 2017 -- what about lastTriggered? new Version(object.field(versionField).asString()), applicationRevisionFromSlime(object.field(revisionField)), object.field(upgradeField).asBool(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 68df16504a8..e5616f025ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -2,10 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; -import com.yahoo.cloud.config.ClusterInfoConfig; -import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.SlimeUtils; @@ -14,7 +11,6 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.zookeeper.ZooKeeperServer; import java.io.IOException; import java.io.UncheckedIOException; @@ -30,7 +26,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Curator backed database for storing working state shared between controller servers. @@ -40,9 +35,6 @@ import java.util.stream.Collectors; */ public class CuratorDb { - /** Use a nonstandard zk port to avoid interfering with connection to the config server zk cluster */ - private static final int zooKeeperPort = 2281; - private static final Logger log = Logger.getLogger(CuratorDb.class.getName()); private static final Path root = Path.fromString("/controller/v1"); @@ -52,9 +44,6 @@ public class CuratorDb { private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); private final JobQueueSerializer jobQueueSerializer = new JobQueueSerializer(); - @SuppressWarnings("unused") // This server is used (only) from the curator instance of this over the network */ - private final ZooKeeperServer zooKeeperServer; - private final Curator curator; /** @@ -63,54 +52,11 @@ public class CuratorDb { */ private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>(); - /** Create a curator db which also set up a ZooKeeper server (such that this instance is both client and server) */ @Inject - public CuratorDb(ClusterInfoConfig clusterInfo) { - this.zooKeeperServer = new ZooKeeperServer(toZookeeperServerConfig(clusterInfo)); - this.curator = new Curator(toConnectionSpec(clusterInfo)); - } - - /** Create a curator db which does not set up a server, using the given Curator instance */ - protected CuratorDb(Curator curator) { - this.zooKeeperServer = null; + public CuratorDb(Curator curator) { this.curator = curator; } - private static ZookeeperServerConfig toZookeeperServerConfig(ClusterInfoConfig clusterInfo) { - ZookeeperServerConfig.Builder b = new ZookeeperServerConfig.Builder(); - b.zooKeeperConfigFile("conf/zookeeper/controller-zookeeper.cfg"); - b.dataDir("var/controller-zookeeper"); - b.clientPort(zooKeeperPort); - b.myidFile("var/controller-zookeeper/myid"); - b.myid(myIndex(clusterInfo)); - - for (ClusterInfoConfig.Services clusterMember : clusterInfo.services()) { - ZookeeperServerConfig.Server.Builder server = new ZookeeperServerConfig.Server.Builder(); - server.id(clusterMember.index()); - server.hostname(clusterMember.hostname()); - server.quorumPort(zooKeeperPort + 1); - server.electionPort(zooKeeperPort + 2); - b.server(server); - } - return new ZookeeperServerConfig(b); - } - - private static Integer myIndex(ClusterInfoConfig clusterInfo) { - String hostname = HostName.getLocalhost(); - return clusterInfo.services().stream() - .filter(service -> service.hostname().equals(hostname)) - .map(ClusterInfoConfig.Services::index) - .findFirst() - .orElseThrow(() -> new IllegalStateException("Unable to find index for this node by hostname '" + - hostname + "'")); - } - - private static String toConnectionSpec(ClusterInfoConfig clusterInfo) { - return clusterInfo.services().stream() - .map(member -> member.hostname() + ":" + zooKeeperPort) - .collect(Collectors.joining(",")); - } - // -------------- Locks -------------------------------------------------- public Lock lock(TenantId id, Duration timeout) { @@ -230,10 +176,6 @@ public class CuratorDb { VersionStatusSerializer serializer = new VersionStatusSerializer(); NestedTransaction transaction = new NestedTransaction(); try { - // TODO: Removes unused data. Remove after October 2017 - if (curator.getData(systemVersionPath()).isPresent()) { - curator.delete(systemVersionPath()); - } curator.set(versionStatusPath(), SlimeUtils.toJsonBytes(serializer.toSlime(status))); } catch (IOException e) { throw new UncheckedIOException("Failed to serialize version status", e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java index 37677a5e393..ab240b9dea9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java @@ -24,9 +24,9 @@ import java.util.stream.Collectors; */ public class MemoryControllerDb extends ControllerDb { - private Map<TenantId, Tenant> tenants = new HashMap<>(); - private Map<String, Application> applications = new HashMap<>(); - private Map<RotationId, ApplicationId> rotationAssignments = new HashMap<>(); + private final Map<TenantId, Tenant> tenants = new HashMap<>(); + private final Map<String, Application> applications = new HashMap<>(); + private final Map<RotationId, ApplicationId> rotationAssignments = new HashMap<>(); @Override public void createTenant(Tenant tenant) { @@ -46,23 +46,14 @@ public class MemoryControllerDb extends ControllerDb { @Override public void deleteTenant(TenantId tenantId) { - Object removed = tenants.remove(tenantId); - if (removed == null) + if (tenants.remove(tenantId) == null) { throw new NotExistsException(tenantId); + } } @Override public Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException { - Optional<Tenant> tenant = Optional.ofNullable(tenants.get(tenantId)); - if(tenant.isPresent()) { - Tenant t_noquota = tenant.get(); - Tenant t_withquota = new Tenant( - t_noquota.getId(), t_noquota.getUserGroup(), t_noquota.getProperty(), - t_noquota.getAthensDomain(), t_noquota.getPropertyId()); - return Optional.of(t_withquota); - } else { - return tenant; - } + return Optional.ofNullable(tenants.get(tenantId)); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 81c3bb963db..6b60b49e1ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -40,6 +41,7 @@ public class VersionStatusSerializer { private static final String versionField = "version"; private static final String failingField = "failing"; private static final String productionField = "production"; + private static final String deployingField = "deploying"; public Slime toSlime(VersionStatus status) { Slime slime = new Slime(); @@ -74,9 +76,10 @@ public class VersionStatusSerializer { object.setString(versionField, statistics.version().toString()); applicationsToSlime(statistics.failing(), object.setArray(failingField)); applicationsToSlime(statistics.production(), object.setArray(productionField)); + applicationsToSlime(statistics.deploying(), object.setArray(deployingField)); } - private void applicationsToSlime(List<ApplicationId> applications, Cursor array) { + private void applicationsToSlime(Collection<ApplicationId> applications, Cursor array) { applications.forEach(application -> array.addString(application.serializedForm())); } @@ -105,7 +108,8 @@ public class VersionStatusSerializer { private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) { return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()), applicationsFromSlime(object.field(failingField)), - applicationsFromSlime(object.field(productionField))); + applicationsFromSlime(object.field(productionField)), + applicationsFromSlime(object.field(deployingField))); } private List<ApplicationId> applicationsFromSlime(Inspector array) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java new file mode 100644 index 00000000000..529acc48cbe --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.yahoo.container.jdisc.HttpResponse; + +/** + * Executes call against config servers and handles discovery requests. Rest URIs in the response are + * rewritten. + * + * @author Haakon Dybdahl + */ +public interface ConfigServerRestExecutor { + HttpResponse handle(ProxyRequest proxyRequest) throws ProxyException; +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java new file mode 100644 index 00000000000..e8b68d0c55a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -0,0 +1,243 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import org.apache.http.Header; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * @author Haakon Dybdahl + */ +@SuppressWarnings("unused") // Injected +public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { + + private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10); + + private final ZoneRegistry zoneRegistry; + + public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry) { + this.zoneRegistry = zoneRegistry; + } + + @Override + public ProxyResponse handle(ProxyRequest proxyRequest) throws ProxyException { + if (proxyRequest.isDiscoveryRequest()) { + return createDiscoveryResponse(proxyRequest); + } + + Environment environment = Environment.from(proxyRequest.getEnvironment()); + RegionName region = RegionName.from(proxyRequest.getRegion()); + + // Make a local copy of the list as we want to manipulate it in case of ping problems. + final List<URI> allServers = new ArrayList<>(zoneRegistry.getConfigServerUris(environment, region)); + + StringBuilder errorBuilder = new StringBuilder(); + if (queueFirstServerIfDown(allServers)) { + errorBuilder.append("Change ordering due to failed ping."); + } + for (URI uri : allServers) { + Optional<ProxyResponse> proxyResponse = proxyCall(uri, proxyRequest, errorBuilder); + if (proxyResponse.isPresent()) { + return proxyResponse.get(); + } + } + // TODO Add logging, for now, experimental and we want to not add more noise. + throw new ProxyException(ErrorResponse.internalServerError("Failed talking to config servers: " + + errorBuilder.toString())); + } + + private static class DiscoveryResponseStructure { + public List<String> uris = new ArrayList<>(); + } + + private ProxyResponse createDiscoveryResponse(ProxyRequest proxyRequest) { + ObjectMapper mapper = new ObjectMapper(); + DiscoveryResponseStructure responseStructure = new DiscoveryResponseStructure(); + + List<Zone> zones = zoneRegistry.zones(); + for (Zone zone : zones) { + if (!"".equals(proxyRequest.getEnvironment()) && + !proxyRequest.getEnvironment().equals(zone.environment().value())) { + continue; + } + responseStructure.uris.add(proxyRequest.getScheme() + "://" + proxyRequest.getControllerPrefix() + + zone.environment().name() + "/" + zone.region().value()); + } + JsonNode node = mapper.valueToTree(responseStructure); + return new ProxyResponse(proxyRequest, node.toString(), 200, Optional.empty(), "application/json"); + } + + private String removeFirstSlashIfAny(String url) { + if (url.startsWith("/")) { + return url.substring(1); + } + return url; + } + + private Optional<ProxyResponse> proxyCall(URI uri, ProxyRequest proxyRequest, StringBuilder errorBuilder) + throws ProxyException { + String fullUri = uri.toString() + removeFirstSlashIfAny(proxyRequest.getConfigServerRequest()); + final HttpRequestBase requestBase = createHttpBaseRequest( + proxyRequest.getMethod(), fullUri, proxyRequest.getData()); + // Empty list of headers to copy for now, add headers when needed, or rewrite logic. + copyHeaders(proxyRequest.getHeaders(), requestBase, new HashSet<>()); + + RequestConfig config = RequestConfig.custom() + .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); + try ( + CloseableHttpClient client = createHttpClient(config); + CloseableHttpResponse response = client.execute(requestBase); + ) { + if (response.getStatusLine().getStatusCode() / 100 == 5) { + errorBuilder.append("Talking to server ").append(uri.getHost()); + errorBuilder.append(", got ").append(response.getStatusLine().getStatusCode()).append(" ") + .append(streamToString(response.getEntity().getContent())).append("\n"); + return Optional.empty(); + } + final Header contentHeader = response.getLastHeader("Content-Type"); + final String contentType; + if (contentHeader != null && contentHeader.getValue() != null && ! contentHeader.getValue().isEmpty()) { + contentType = contentHeader.getValue().replace("; charset=UTF-8",""); + } else { + contentType = "application/json"; + } + return Optional.of(new ProxyResponse( + proxyRequest, + streamToString(response.getEntity().getContent()), + response.getStatusLine().getStatusCode(), + Optional.of(uri), + contentType)); + + // Send response back + } catch (IOException|RuntimeException e) { + errorBuilder.append("Talking to server ").append(uri.getHost()); + errorBuilder.append(" got exception ").append(e.getMessage()); + return Optional.empty(); + } + } + + private HttpRequestBase createHttpBaseRequest(String method, String uri, InputStream data) throws ProxyException { + Method enumMethod = Method.valueOf(method); + switch (enumMethod) { + case GET: + return new HttpGet(uri); + case POST: + HttpPost post = new HttpPost(uri); + if (data != null) { + post.setEntity(new InputStreamEntity(data)); + } + return post; + case PUT: + HttpPut put = new HttpPut(uri); + if (data != null) { + put.setEntity(new InputStreamEntity(data)); + } + return put; + case DELETE: + return new HttpDelete(uri); + case PATCH: + HttpPatch patch = new HttpPatch(uri); + if (data != null) { + patch.setEntity(new InputStreamEntity(data)); + } + return patch; + default: + throw new ProxyException(ErrorResponse.methodNotAllowed("Will not proxy such calls.")); + } + } + + private void copyHeaders(Map<String, List<String>> headers, HttpRequestBase toRequest, Set<String> headersToCopy) { + for (Map.Entry<String, List<String>> headerEntry : headers.entrySet()) { + for (String value : headerEntry.getValue()) { + if (headersToCopy.contains(value)) { + toRequest.addHeader(headerEntry.getKey(), value); + } + } + } + } + + public static String streamToString(final InputStream inputStream) throws IOException { + final StringBuilder out = new StringBuilder(); + while (true) { + byte[] bytesFromStream = IOUtils.readBytes(inputStream, 1024); + if (bytesFromStream.length == 0) { + return out.toString(); + } + out.append(new String(bytesFromStream, StandardCharsets.UTF_8)); + } + } + + /** + * During upgrade, one server can be down, this is normal. Therefor we do a quick ping on the first server, + * if it is not responding, we try the other servers first. False positive/negatives are not critical, + * but will increase latency to some extent. + */ + private boolean queueFirstServerIfDown(List<URI> allServers) { + if (allServers.size() < 2) { + return false; + } + URI uri = allServers.get(0); + HttpGet httpget = new HttpGet(uri); + + int timeout = 500; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout).build(); + try ( + CloseableHttpClient client = createHttpClient(config); + CloseableHttpResponse response = client.execute(httpget); + + ) { + if (response.getStatusLine().getStatusCode() == 200) { + return false; + } + + } catch (IOException e) { + // We ignore this, if server is restarting this might happen. + } + // Some error happened, move this server to the back. The other servers should be running. + allServers.remove(0); + allServers.add(uri); + return true; + } + + private static CloseableHttpClient createHttpClient(RequestConfig config) { + return HttpClientBuilder.create() + .setUserAgent("config-server-client") + .setDefaultRequestConfig(config) + .build(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java new file mode 100644 index 00000000000..3673c0227a3 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; + +/** + * Class for generating error responses. + * + * @author Haakon Dybdahl + */ +public class ErrorResponse extends HttpResponse { + + private final Slime slime = new Slime(); + public final String message; + + public ErrorResponse(int code, String errorType, String message) { + super(code); + this.message = message; + Cursor root = slime.setObject(); + root.setString("error-code", errorType); + root.setString("message", message); + } + + public enum errorCodes { + NOT_FOUND, + BAD_REQUEST, + METHOD_NOT_ALLOWED, + INTERNAL_SERVER_ERROR, + + } + + public static ErrorResponse notFoundError(String message) { + return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message); + } + + public static ErrorResponse internalServerError(String message) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message); + } + + public static ErrorResponse badRequest(String message) { + return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message); + } + + public static ErrorResponse methodNotAllowed(String message) { + return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java new file mode 100644 index 00000000000..aa828bc0c83 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +/** + * Exceptions related to proxying calls to config servers. + * + * @author Haakon Dybdahl + */ +public class ProxyException extends Exception { + public final ErrorResponse errorResponse; + + public ProxyException(ErrorResponse errorResponse) { + super(errorResponse.message); + this.errorResponse = errorResponse; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java new file mode 100644 index 00000000000..6854d583222 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -0,0 +1,119 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.net.HostName; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLDecoder; +import java.util.List; +import java.util.Map; + +/** + * Keeping information about the calls that are being proxied. + * A request is of form /zone/v2/[environment]/[region]/[config-server-path] + * + * @author Haakon Dybdahl + */ +public class ProxyRequest { + + private final String environment; + private final String region; + private final String configServerRequest; + private final InputStream requestData; + private final Map<String, List<String>> headers; + private final String method; + private final String controllerPrefix; + private final String scheme; + + /** + * The constructor calls exception if the request is invalid. + * + * @param request the request from the jdisc framework. + * @param pathPrefix the path prefix of the proxy. + * @throws ProxyException on errors + */ + public ProxyRequest(HttpRequest request, String pathPrefix) throws ProxyException, IOException { + this(request.getUri(), request.getJDiscRequest().headers(), request.getData(), request.getMethod().name(), + pathPrefix); + } + + ProxyRequest(URI requestUri, Map<String, List<String>> headers, InputStream body, String method, + String pathPrefix) throws ProxyException, IOException { + if (requestUri == null) { + throw new ProxyException(ErrorResponse.badRequest("Request not set.")); + } + final String path = URLDecoder.decode(requestUri.getPath(),"UTF-8"); + if (! path.startsWith(pathPrefix)) { + // This has to be caused by wrong mapping of path in services.xml. + throw new ProxyException(ErrorResponse.notFoundError("Request not starting with " + pathPrefix)); + } + final String uriNoPrefix = path.replaceFirst(pathPrefix, "") + + (requestUri.getRawQuery() == null ? "" : "?" + requestUri.getRawQuery()); + + final String[] parts = uriNoPrefix.split("/"); + + this.environment = parts.length > 0 ? parts[0] : ""; + this.region = parts.length > 1 ? parts[1] : ""; + this.configServerRequest = parts.length > 2 ? uriNoPrefix.replace(environment + "/" + region, "") : ""; + this.requestData = body; + this.headers = headers; + this.method = method; + + String hostPort = headers.containsKey("host") + ? headers.get("host").get(0) + : HostName.getLocalhost() + ":" + requestUri.getPort(); + StringBuilder prefix = new StringBuilder(hostPort + pathPrefix); + if (! environment.isEmpty()) { + prefix.append(environment).append("/").append(region); + } + + this.controllerPrefix = prefix.toString(); + this.scheme = requestUri.getScheme(); + } + + /** + * A discovery query lists environments and regions. + */ + public boolean isDiscoveryRequest() { + return region.isEmpty(); + } + + public String getRegion() { + return region; + } + + public String getEnvironment() { + return environment; + } + + public String getConfigServerRequest() { + return configServerRequest; + } + + public InputStream getData() { + return requestData; + } + + @Override + public String toString() { + return "[ region: " + region + " env: " + environment + " request: " + configServerRequest + "]"; + } + + public Map<String, List<String>> getHeaders() { + return headers; + } + + public String getMethod() { + return method; + } + + public String getControllerPrefix() { + return controllerPrefix; + } + + public String getScheme() { return scheme; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java new file mode 100644 index 00000000000..3f878740ff0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.yahoo.container.jdisc.HttpResponse; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * Response class that also rewrites URL from config server. + * + * @author Haakon Dybdahl + */ +public class ProxyResponse extends HttpResponse { + + private final String bodyResponseRewritten; + private final String contentType; + + public ProxyResponse( + ProxyRequest controllerRequest, + String bodyResponse, + int statusResponse, + Optional<URI> configServer, + String contentType) { + super(statusResponse); + this.contentType = contentType; + + if (! configServer.isPresent() || controllerRequest.getControllerPrefix().isEmpty()) { + bodyResponseRewritten = bodyResponse; + return; + } + + final String configServerPrefix; + final String controllerRequestPrefix; + try { + configServerPrefix = new URIBuilder() + .setScheme(configServer.get().getScheme()) + .setHost(configServer.get().getHost()) + .setPort(configServer.get().getPort()) + .build().toString(); + controllerRequestPrefix = new URIBuilder() + .setScheme(controllerRequest.getScheme()) + // controller prefix is more than host, so it is a bit hackish, but verified by tests. + .setHost(controllerRequest.getControllerPrefix()) + .build().toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + bodyResponseRewritten = bodyResponse.replace(configServerPrefix, controllerRequestPrefix); + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(bodyResponseRewritten.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { return contentType; } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java new file mode 100644 index 00000000000..f6c300268a2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.proxy; + +/** + * @author Haakon Dybdahl + */ +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java index c8c027d91c9..e9db4f9b717 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java @@ -76,8 +76,8 @@ public class Path { StringBuilder rest = new StringBuilder(); for (int i = specElements.length; i < this.elements.length; i++) rest.append(elements[i]).append("/"); - if ( ! pathString.endsWith("/")) - rest.setLength(rest.length() -1); + if ( ! pathString.endsWith("/") && rest.length() > 0) + rest.setLength(rest.length() - 1); this.rest = rest.toString(); } @@ -98,9 +98,6 @@ public class Path { */ public String getRest() { return rest; } - /** Returns this path as a string */ - public String asString() { return pathString; } - @Override public String toString() { return "path '" + Arrays.stream(elements).collect(Collectors.joining("/")) + "'"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index a259e221a1e..d7324450d4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -50,9 +51,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -62,7 +63,6 @@ import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; @@ -169,7 +169,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/cookiefreshness")) return cookieFreshness(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), path, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/converge")) return waitForConvergence(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -222,10 +222,19 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.headers().put("Allow", "GET,PUT,POST,DELETE,OPTIONS"); return response; } - + + private HttpResponse recursiveRoot(HttpRequest request) { + Slime slime = new Slime(); + Cursor tenantArray = slime.setArray(); + for (Tenant tenant : controller.tenants().asList()) + toSlime(tenantArray.addObject(), tenant, request, true); + return new SlimeJsonResponse(slime); + } + private HttpResponse root(HttpRequest request) { - return new ResourceResponse(request, "user", "tenant", "tenant-pipeline", "athensDomain", - "property", "cookiefreshness"); + return recurseOverTenants(request) + ? recursiveRoot(request) + : new ResourceResponse(request, "user", "tenant", "tenant-pipeline", "athensDomain", "property", "cookiefreshness"); } private HttpResponse authenticatedUser(HttpRequest request) { @@ -310,20 +319,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse tenant(Tenant tenant, HttpRequest request, boolean listApplications) { - Slime tenantSlime = toSlime(tenant, request, listApplications); - tenant.getPropertyId().ifPresent(propertyId -> { - try { - toSlime(tenantSlime.get(), - controller.organization().propertyUri(propertyId), - controller.organization().contactsUri(propertyId), - controller.organization().issueCreationUri(propertyId), - controller.organization().contactsFor(propertyId)); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Error fetching property info for " + tenant + " with propertyId " + propertyId, e); - } - }); - return new SlimeJsonResponse(tenantSlime); + Slime slime = new Slime(); + toSlime(slime.setObject(), tenant, request, listApplications); + return new SlimeJsonResponse(slime); } private HttpResponse applications(String tenantName, HttpRequest request) { @@ -335,18 +333,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse application(String tenantName, String applicationName, Path path, HttpRequest request) { - Slime slime = new Slime(); - Cursor response = slime.setObject(); - - com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse application(String tenantName, String applicationName, HttpRequest request) { + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); Application application = controller.applications().get(applicationId) .orElseThrow(() -> new NotExistsException(applicationId + " not found")); - + + Slime slime = new Slime(); + toSlime(slime.setObject(), application, request); + return new SlimeJsonResponse(slime); + } + + private void toSlime(Cursor object, Application application, HttpRequest request) { + object.setString("application", application.id().application().value()); + object.setString("instance", application.id().instance().value()); // Currently deploying change if (application.deploying().isPresent()) { - Cursor deployingObject = response.setObject("deploying"); + Cursor deployingObject = object.setObject("deploying"); if (application.deploying().get() instanceof Change.VersionChange) deployingObject.setString("version", ((Change.VersionChange)application.deploying().get()).version().toString()); else if (((Change.ApplicationChange)application.deploying().get()).revision().isPresent()) @@ -354,14 +357,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } // Jobs sorted according to deployment spec - Map<DeploymentJobs.JobType, JobStatus> jobStatus = controller.applications().deploymentTrigger() + List<JobStatus> jobStatus = controller.applications().deploymentTrigger() .deploymentOrder() - .sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus()); + .sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus().values()); - Cursor deploymentsArray = response.setArray("deploymentJobs"); - for (JobStatus job : jobStatus.values()) { - Cursor jobObject = deploymentsArray.addObject(); - jobObject.setString("type", job.type().id()); + Cursor deploymentsArray = object.setArray("deploymentJobs"); + for (JobStatus job : jobStatus) { + Cursor jobObject = deploymentsArray.addObject(); + jobObject.setString("type", job.type().jobName()); jobObject.setBool("success", job.isSuccess()); job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered"))); @@ -371,47 +374,47 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } // Compile version. The version that should be used when building an application - response.setString("compileVersion", application.compileVersion(controller).toFullString()); + object.setString("compileVersion", application.oldestDeployedVersion().orElse(controller.systemVersion()).toFullString()); // Rotations - Cursor globalRotationsArray = response.setArray("globalRotations"); - Set<URI> rotations = controller.getRotationUris(applicationId); + Cursor globalRotationsArray = object.setArray("globalRotations"); + Set<URI> rotations = controller.getRotationUris(application.id()); Map<String, RotationStatus> rotationHealthStatus = rotations.isEmpty() ? Collections.emptyMap() : controller.getHealthStatus(rotations.iterator().next().getHost()); for (URI rotation : rotations) globalRotationsArray.addString(rotation.toString()); // Deployments sorted according to deployment spec - Map<Zone, Deployment> deployments = controller.applications().deploymentTrigger() + List<Deployment> deployments = controller.applications().deploymentTrigger() .deploymentOrder() - .sortBy(application.deploymentSpec().zones(), application.deployments()); - Cursor instancesArray = response.setArray("instances"); - for (Deployment deployment : deployments.values()) { + .sortBy(application.deploymentSpec().zones(), application.deployments().values()); + Cursor instancesArray = object.setArray("instances"); + for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); + deploymentObject.setString("environment", deployment.zone().environment().value()); deploymentObject.setString("region", deployment.zone().region().value()); deploymentObject.setString("instance", application.id().instance().value()); // pointless if ( ! rotations.isEmpty()) setRotationStatus(deployment, rotationHealthStatus, deploymentObject); - deploymentObject.setString("url", withPath(path.asString() + - "/environment/" + deployment.zone().environment().value() + - "/region/" + deployment.zone().region().value() + - "/instance/" + application.id().instance().value(), - request.getUri()).toString()); + + if (recurseOverDeployments(request)) // List full deployment information when recursive. + toSlime(deploymentObject, new DeploymentId(application.id(), deployment.zone()), deployment, request); + else + deploymentObject.setString("url", withPath(request.getUri().getPath() + + "/environment/" + deployment.zone().environment().value() + + "/region/" + deployment.zone().region().value() + + "/instance/" + application.id().instance().value(), + request.getUri()).toString()); } - + // Metrics - try { - MetricsService.ApplicationMetrics metrics = controller.metricsService().getApplicationMetrics(applicationId); - Cursor metricsObject = response.setObject("metrics"); - metricsObject.setDouble("queryServiceQuality", metrics.queryServiceQuality()); - metricsObject.setDouble("writeServiceQuality", metrics.writeServiceQuality()); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed getting Yamas metrics", Exceptions.toMessageString(e)); - } + Cursor metricsObject = object.setObject("metrics"); + metricsObject.setDouble("queryServiceQuality", application.metrics().queryServiceQuality()); + metricsObject.setDouble("writeServiceQuality", application.metrics().writeServiceQuality()); - return new SlimeJsonResponse(slime); + application.ownershipIssueId().ifPresent(issueId -> object.setString("ownershipIssueId", issueId.value())); + application.deploymentJobs().issueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value())); } private HttpResponse deployment(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { @@ -426,21 +429,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (deployment == null) throw new NotExistsException(application + " is not deployed in " + deploymentId.zone()); - Optional<InstanceEndpoints> deploymentEndpoints = controller.applications().getDeploymentEndpoints(deploymentId); - Slime slime = new Slime(); - Cursor response = slime.setObject(); + toSlime(slime.setObject(), deploymentId, deployment, request); + return new SlimeJsonResponse(slime); + } + + private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { + + Optional<InstanceEndpoints> deploymentEndpoints = controller.applications().getDeploymentEndpoints(deploymentId); Cursor serviceUrlArray = response.setArray("serviceUrls"); if (deploymentEndpoints.isPresent()) { for (URI uri : deploymentEndpoints.get().getContainerEndpoints()) serviceUrlArray.addString(uri.toString()); } - response.setString("nodes", withPath("/zone/v2/" + environment + "/" + region + "/nodes/v2/node/?&recursive=true&application=" + tenantName + "." + applicationName + "." + instanceName, request.getUri()).toString()); + response.setString("nodes", withPath("/zone/v2/" + deploymentId.zone().environment() + "/" + deploymentId.zone().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); - Environment env = Environment.from(environment); - RegionName regionName = RegionName.from(region); - URI elkUrl = controller.getElkUri(env, regionName, deploymentId); + URI elkUrl = controller.getElkUri(deploymentId); if (elkUrl != null) response.setString("elkUrl", elkUrl.toString()); @@ -448,10 +453,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("version", deployment.version().toFullString()); response.setString("revision", deployment.revision().id()); response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli()); - Optional<Duration> deploymentTimeToLive = controller.zoneRegistry().getDeploymentTimeToLive(Environment.from(environment), RegionName.from(region)); + Optional<Duration> deploymentTimeToLive = controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zone().environment(), deploymentId.zone().region()); deploymentTimeToLive.ifPresent(duration -> response.setLong("expiryTimeEpochMs", deployment.at().plus(duration).toEpochMilli())); - application.deploymentJobs().projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); + controller.applications().get(deploymentId.applicationId()).flatMap(application -> application.deploymentJobs().projectId()) + .ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); sourceRevisionToSlime(deployment.revision().source(), response); // Cost @@ -467,8 +473,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metricsObject.setDouble("documentCount", metrics.documentCount()); metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); - - return new SlimeJsonResponse(slime); } private void toSlime(ApplicationRevision revision, Cursor object) { @@ -575,7 +579,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)), - new com.yahoo.config.provision.ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), + new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)), request.getUri()); response.setResponse(applicationView); @@ -585,7 +589,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)), - new com.yahoo.config.provision.ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), + new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)), request.getUri()); response.setResponse(result, serviceName, restPath); @@ -619,8 +623,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } case OPSDB: { UserGroup userGroup = new UserGroup(mandatory("userGroup", requestData).asString()); - updatedTenant = Tenant.createOpsDbTenant(new TenantId(tenantName), - userGroup, + updatedTenant = Tenant.createOpsDbTenant(new TenantId(tenantName), + userGroup, new Property(mandatory("property", requestData).asString()), optional("propertyId", requestData).map(PropertyId::new)); throwIfNotSuperUserOrPartOfOpsDbGroup(userGroup, request); @@ -630,7 +634,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { case ATHENS: { if (requestData.field("userGroup").valid()) throw new BadRequestException("Cannot set OpsDB user group to Athens tenant"); - updatedTenant = Tenant.createAthensTenant(new TenantId(tenantName), + updatedTenant = Tenant.createAthensTenant(new TenantId(tenantName), new AthenzDomain(mandatory("athensDomain", requestData).asString()), new Property(mandatory("property", requestData).asString()), optional("propertyId", requestData).map(PropertyId::new)); @@ -659,7 +663,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throwIfNotSuperUserOrPartOfOpsDbGroup(new UserGroup(mandatory("userGroup", requestData).asString()), request); if (tenant.isAthensTenant()) throwIfNotAthenzDomainAdmin(new AthenzDomain(mandatory("athensDomain", requestData).asString()), request); - + controller.tenants().addTenant(tenant, authorizer.getNToken(request)); return tenant(tenant, request, true); } @@ -675,7 +679,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throwIfNotAthenzDomainAdmin(tenantDomain, request); NToken nToken = authorizer.getNToken(request) .orElseThrow(() -> - new BadRequestException("The NToken for a domain admin is required to migrate tenant to Athens")); + new BadRequestException("The NToken for a domain admin is required to migrate tenant to Athens")); Tenant tenant = controller.tenants().migrateTenantToAthenz(tenantid, tenantDomain, propertyId, property, nToken); return tenant(tenant, request, true); } @@ -684,7 +688,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { authorizer.throwIfUnauthorized(new TenantId(tenantName), request); Application application; try { - application = controller.applications().createApplication(com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"), authorizer.getNToken(request)); + application = controller.applications().createApplication(ApplicationId.from(tenantName, applicationName, "default"), authorizer.getNToken(request)); } catch (ZmsException e) { // TODO: Push conversion down if (e.getCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) @@ -700,36 +704,35 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Trigger deployment of the last built application package, on a given version */ private HttpResponse deploy(String tenantName, String applicationName, HttpRequest request) { + Version version = decideDeployVersion(request); + if ( ! systemHasVersion(version)) + throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + + "Version is not active in this system. " + + "Active versions: " + controller.versionStatus().versions()); + ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); - try (Lock lock = controller.applications().lock(id)) { - Application application = controller.applications().require(id); + controller.applications().lockedOrThrow(id, application -> { if (application.deploying().isPresent()) throw new IllegalArgumentException("Can not start a deployment of " + application + " at this time: " + - application.deploying().get() + " is in progress"); - - Version version = decideDeployVersion(request); - if ( ! systemHasVersion(version)) - throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + - "Version is not active in this system. " + - "Active versions: " + controller.versionStatus().versions()); + application.deploying().get() + " is in progress"); controller.applications().deploymentTrigger().triggerChange(application.id(), new Change.VersionChange(version)); - return new MessageResponse("Triggered deployment of " + application + " on version " + version); - } + }); + return new MessageResponse("Triggered deployment of application '" + id + "' on version " + version); } /** Cancel any ongoing change for given application */ private HttpResponse cancelDeploy(String tenantName, String applicationName) { ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); - try (Lock lock = controller.applications().lock(id)) { - Application application = controller.applications().require(id); - Optional<Change> change = application.deploying(); - if (!change.isPresent()) { - return new MessageResponse("No deployment in progress for " + application + " at this time"); - } - controller.applications().deploymentTrigger().cancelChange(id); - return new MessageResponse("Cancelled " + change.get() + " for " + application); - } + Application application = controller.applications().require(id); + Optional<Change> change = application.deploying(); + if ( ! change.isPresent()) + return new MessageResponse("No deployment in progress for " + application + " at this time"); + + controller.applications().lockedOrThrow(id, lockedApplication -> + controller.applications().deploymentTrigger().cancelChange(id)); + + return new MessageResponse("Cancelled " + change.get() + " for " + application); } /** Schedule restart of deployment, or specific host in a deployment */ @@ -789,10 +792,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { DeployOptions deployOptionsJsonClass = new DeployOptions(screwdriverBuildJobFromSlime(deployOptions.field("screwdriverBuildJob")), optional("vespaVersion", deployOptions).map(Version::new), deployOptions.field("ignoreValidationErrors").asBool(), - deployOptions.field("deployCurrentVersion").asBool()); - ActivateResult result = controller.applications().deployApplication(applicationId, - zone, - new ApplicationPackage(dataParts.get("applicationZip")), + deployOptions.field("deployCurrentVersion").asBool()); + ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get("applicationZip")); + controller.applications().validate(applicationPackage.deploymentSpec()); + ActivateResult result = controller.applications().deployApplication(applicationId, + zone, + applicationPackage, deployOptionsJsonClass); return new SlimeJsonResponse(toSlime(result, dataParts.get("applicationZip").length)); } @@ -811,10 +816,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { authorizer.throwIfUnauthorized(new TenantId(tenantName), request); - com.yahoo.config.provision.ApplicationId id = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"); - Application deleted = controller.applications().deleteApplication(id, authorizer.getNToken(request)); - if (deleted == null) - return ErrorResponse.notFoundError("Could not delete application '" + id + "': Application not found"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + controller.applications().deleteApplication(id, authorizer.getNToken(request)); return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead } @@ -874,7 +877,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return authorizer.getPrincipalIfAny(request).map(Principal::getName); } - private void toSlime(Tenant tenant, Cursor object, HttpRequest request, boolean listApplications) { + private void toSlime(Cursor object, Tenant tenant, HttpRequest request, boolean listApplications) { + object.setString("tenant", tenant.getId().id()); object.setString("type", tenant.tenantType().name()); tenant.getAthensDomain().ifPresent(a -> object.setString("athensDomain", a.id())); tenant.getProperty().ifPresent(p -> object.setString("property", p.id())); @@ -883,10 +887,31 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor applicationArray = object.setArray("applications"); if (listApplications) { // This cludge is needed because we call this after deleting the tenant. As this call makes another tenant lookup it will fail. TODO is to support lookup on tenant for (Application application : controller.applications().asList(TenantName.from(tenant.getId().id()))) { - if (application.id().instance().isDefault()) // TODO: Skip non-default applications until supported properly - toSlime(application, applicationArray.addObject(), request); + if (application.id().instance().isDefault()) {// TODO: Skip non-default applications until supported properly + if (recurseOverApplications(request)) + toSlime(applicationArray.addObject(), application, request); + else + toSlime(application, applicationArray.addObject(), request); + } } } + tenant.getPropertyId().ifPresent(propertyId -> { + try { + object.setString("propertyUrl", controller.organization().propertyUri(propertyId).toString()); + object.setString("contactsUrl", controller.organization().contactsUri(propertyId).toString()); + object.setString("issueCreationUrl", controller.organization().issueCreationUri(propertyId).toString()); + Cursor lists = object.setArray("contacts"); + for (List<? extends User> contactList : controller.organization().contactsFor(propertyId)) { + Cursor list = lists.addArray(); + for (User contact : contactList) + list.addString(contact.displayName()); + } + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Error fetching property info for " + tenant + " with propertyId " + propertyId + ": " + + Exceptions.toMessageString(e)); + } + }); } // A tenant has different content when in a list ... antipattern, but not solvable before application/v5 @@ -989,24 +1014,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return Joiner.on("/").join(elements); } - private Slime toSlime(Tenant tenant, HttpRequest request, boolean listApplications) { - Slime slime = new Slime(); - toSlime(tenant, slime.setObject(), request, listApplications); - return slime; - } - - private void toSlime(Cursor root, URI propertyUri, URI contactsUri, URI issueCreationUri, List<? extends List<? extends User>> contacts) { - root.setString("propertyUrl", propertyUri.toString()); - root.setString("contactsUrl", contactsUri.toString()); - root.setString("issueCreationUrl", issueCreationUri.toString()); - Cursor lists = root.setArray("contacts"); - for (List<? extends User> contactList : contacts) { - Cursor list = lists.addArray(); - for (User contact : contactList) - list.addString(contact.displayName()); - } - } - private void toSlime(Application application, Cursor object, HttpRequest request) { object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); @@ -1152,4 +1159,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return name; } + + private static boolean recurseOverTenants(HttpRequest request) { + return recurseOverApplications(request) || "tenant".equals(request.getProperty("recursive")); + } + + private static boolean recurseOverApplications(HttpRequest request) { + return recurseOverDeployments(request) || "application".equals(request.getProperty("recursive")); + } + + private static boolean recurseOverDeployments(HttpRequest request) { + return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive")); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 8a5f1e4639a..27b219cd892 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; @@ -11,6 +12,8 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.JobList; +import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; @@ -19,17 +22,20 @@ import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; import com.yahoo.yolean.Exceptions; -import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; +import static java.util.Comparator.comparing; + /** * This implements the deployment/v1 API which provides information about the status of Vespa platform and * application deployments. - * + * * @author bratseth */ +@SuppressWarnings("unused") // Injected public class DeploymentApiHandler extends LoggingRequestHandler { private final Controller controller; @@ -56,7 +62,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); } } - + private HttpResponse handleGET(HttpRequest request) { Path path = new Path(request.getUri().getPath()); if (path.matches("/deployment/v1/")) return root(request); @@ -70,7 +76,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { response.headers().put("Allow", "GET,OPTIONS"); return response; } - + private HttpResponse root(HttpRequest request) { Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -83,7 +89,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { versionObject.setLong("date", version.releasedAt().toEpochMilli()); versionObject.setBool("controllerVersion", version.isSelfVersion()); versionObject.setBool("systemVersion", version.isCurrentSystemVersion()); - + Cursor configServerArray = versionObject.setArray("configServers"); for (String configServerHostnames : version.configServerHostnames()) { Cursor configServerObject = configServerArray.addObject(); @@ -92,29 +98,42 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor failingArray = versionObject.setArray("failingApplications"); for (ApplicationId id : version.statistics().failing()) { - Optional<Application> application = controller.applications().get(id); - if ( ! application.isPresent()) continue; // deleted just now - - Instant failingSince = application.get().deploymentJobs().failingSince(); - if (failingSince == null) continue; // started working just now - - Cursor applicationObject = failingArray.addObject(); - toSlime(application.get(), applicationObject, request); - applicationObject.setLong("failingSince", failingSince.toEpochMilli()); - + controller.applications().get(id).ifPresent(application -> { + firstFailingOn(version.versionNumber(), application).ifPresent(firstFailing -> { + Cursor applicationObject = failingArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setString("failing", firstFailing.type().jobName()); + }); + }); } Cursor productionArray = versionObject.setArray("productionApplications"); for (ApplicationId id : version.statistics().production()) { - Optional<Application> application = controller.applications().get(id); - if ( ! application.isPresent()) continue; // deleted just now - toSlime(application.get(), productionArray.addObject(), request); + controller.applications().get(id).ifPresent(application -> { + int successes = productionSuccessesFor(version.versionNumber(), application); + if (successes == 0) return; // Just upgraded to a newer version. + Cursor applicationObject = productionArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setLong("productionJobs", productionJobsFor(application)); + applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), application)); + }); + } + + Cursor runningArray = versionObject.setArray("deployingApplications"); + for (ApplicationId id : version.statistics().deploying()) { + controller.applications().get(id).ifPresent(application -> { + lastDeployingTo(version.versionNumber(), application).ifPresent(lastDeploying -> { + Cursor applicationObject = runningArray.addObject(); + toSlime(applicationObject, application, request); + applicationObject.setString("running", lastDeploying.type().jobName()); + }); + }); } } return new SlimeJsonResponse(slime); } - private void toSlime(Application application, Cursor object, HttpRequest request) { + private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); @@ -132,4 +151,40 @@ public class DeploymentApiHandler extends LoggingRequestHandler { return upgradePolicy.name(); } + // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus + + /** The first upgrade job to fail on this version, for this application */ + private Optional<JobStatus> firstFailingOn(Version version, Application application) { + return JobList.from(application) + .failing() + .not().failingApplicationChange() + .not().failingBecause(outOfCapacity) + .lastCompleted().on(version) + .asList().stream() + .min(comparing(job -> job.lastCompleted().get().at())); + } + + /** The number of production jobs for this application */ + private int productionJobsFor(Application application) { + return JobList.from(application) + .production() + .size(); + } + + /** The number of production jobs with last success on the given version, for this application */ + private int productionSuccessesFor(Version version, Application application) { + return JobList.from(application) + .production() + .lastSuccess().on(version) + .size(); + } + + /** The last triggered upgrade to this version, for this application */ + private Optional<JobStatus> lastDeployingTo(Version version, Application application) { + return JobList.from(application) + .upgrading() + .asList().stream() + .max(comparing(job -> job.lastTriggered().get().at())); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java index aea59c16cd5..8a539720a21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java @@ -3,16 +3,15 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.google.common.collect.ImmutableMap; +import java.time.Duration; import java.util.Map; -import static java.util.concurrent.TimeUnit.DAYS; - /** * @author gv */ public interface AccessControlHeaders { - String CORS_PREFLIGHT_REQUEST_CACHE_TTL = Long.toString(DAYS.toSeconds(7)); + String CORS_PREFLIGHT_REQUEST_CACHE_TTL = Long.toString(Duration.ofDays(7).getSeconds()); String ALLOW_ORIGIN_HEADER = "Access-Control-Allow-Origin"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java index 850130ca970..6073307bafa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java @@ -19,6 +19,7 @@ import java.security.Principal; */ @After("BouncerFilter") @Provides("SecurityContext") +@SuppressWarnings("unused") // Injected public class CreateSecurityContextFilter implements SecurityRequestFilter { @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 3dbff0b4aa3..e350b98adb9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -103,26 +103,26 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { } private HttpResponse trigger(HttpRequest request, String tenantName, String applicationName) { + JobType jobType = Optional.of(asString(request.getData())) + .filter(s -> !s.isEmpty()) + .map(JobType::fromJobName) + .orElse(JobType.component); + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); - try (Lock lock = controller.applications().lock(applicationId)) { - LockedApplication application = controller.applications().require(applicationId, lock); - JobType jobType = Optional.of(asString(request.getData())) - .filter(s -> !s.isEmpty()) - .map(JobType::fromId) - .orElse(JobType.component); + controller.applications().lockedOrThrow(applicationId, application -> { // Since this is a manual operation we likely want it to trigger as soon as possible so we add it at to the // front of the queue application = controller.applications().deploymentTrigger().triggerAllowParallel( jobType, application, true, true, - "Triggered from the screwdriver/v1 web service" + "Triggered from screwdriver/v1" ); controller.applications().store(application); + }); - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString("message", "Triggered " + jobType.id() + " for " + applicationId); - return new SlimeJsonResponse(slime); - } + Slime slime = new Slime(); + Cursor cursor = slime.setObject(); + cursor.setString("message", "Triggered " + jobType.jobName() + " for " + applicationId); + return new SlimeJsonResponse(slime); } private HttpResponse vespaVersion() { @@ -174,7 +174,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { report.field("tenant").asString(), report.field("application").asString(), report.field("instance").asString()), - JobType.fromId(report.field("jobName").asString()), + JobType.fromJobName(report.field("jobName").asString()), report.field("projectId").asLong(), report.field("buildNumber").asLong(), jobError diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java new file mode 100644 index 00000000000..3a3fd445bcf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -0,0 +1,131 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.zone.v1; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.yolean.Exceptions; + +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * REST API that provides information about Hosted Vespa zones (version 1) + * + * @author mpolden + */ +@SuppressWarnings("unused") +public class ZoneApiHandler extends LoggingRequestHandler { + + private final ZoneRegistry zoneRegistry; + + public ZoneApiHandler(Executor executor, AccessLog accessLog, ZoneRegistry zoneRegistry) { + super(executor, accessLog); + this.zoneRegistry = zoneRegistry; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: + return get(request); + default: + return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + } + } catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse get(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/zone/v1")) { + return root(request); + } + if (path.matches("/zone/v1/environment/{environment}")) { + return environment(request, Environment.from(path.get("environment"))); + } + if (path.matches("/zone/v1/environment/{environment}/default")) { + return defaultRegion(request, Environment.from(path.get("environment"))); + } + return notFound(path); + } + + private HttpResponse root(HttpRequest request) { + List<Environment> environments = zoneRegistry.zones().stream() + .map(Zone::environment) + .distinct() + .sorted(Comparator.comparing(Environment::value)) + .collect(Collectors.toList()); + Slime slime = new Slime(); + Cursor root = slime.setArray(); + environments.forEach(environment -> { + Cursor object = root.addObject(); + object.setString("name", environment.value()); + // Returning /zone/v2 is a bit strange, but that's what the original Jersey implementation did + object.setString("url", request.getUri() + .resolve("/zone/v2/environment/") + .resolve(environment.value()) + .toString()); + }); + return new SlimeJsonResponse(slime); + } + + private HttpResponse environment(HttpRequest request, Environment environment) { + List<Zone> zones = zoneRegistry.zones().stream() + .filter(zone -> zone.environment() == environment) + .collect(Collectors.toList()); + Slime slime = new Slime(); + Cursor root = slime.setArray(); + zones.forEach(zone -> { + Cursor object = root.addObject(); + object.setString("name", zone.region().value()); + object.setString("url", request.getUri() + .resolve("/zone/v2/environment/") + .resolve(environment.value() + "/") + .resolve("region/") + .resolve(zone.region().value()) + .toString()); + }); + return new SlimeJsonResponse(slime); + } + + private HttpResponse defaultRegion(HttpRequest request, Environment environment) { + RegionName region = zoneRegistry.getDefaultRegion(environment) + .orElseThrow(() -> new IllegalArgumentException( + "No default region for environment: " + environment + )); + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("name", region.value()); + root.setString("url", request.getUri() + .resolve("/zone/v2/environment/") + .resolve(environment.value() + "/") + .resolve("region/") + .resolve(region.value()) + .toString()); + return new SlimeJsonResponse(slime); + } + + private HttpResponse notFound(Path path) { + return ErrorResponse.notFoundError("Nothing at " + path); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java new file mode 100644 index 00000000000..7793548766e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +package com.yahoo.vespa.hosted.controller.restapi.zone.v1; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java new file mode 100644 index 00000000000..529b2b25785 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -0,0 +1,116 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.zone.v2; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; +import com.yahoo.vespa.hosted.controller.proxy.ProxyException; +import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +/** + * REST API for proxying requests to config servers in a given zone (version 2). + * + * This API does something completely different from /zone/v1, but such is the world. + * + * @author mpolden + */ +@SuppressWarnings("unused") +public class ZoneApiHandler extends LoggingRequestHandler { + + private final ZoneRegistry zoneRegistry; + private final ConfigServerRestExecutor proxy; + + public ZoneApiHandler(Executor executor, AccessLog accessLog, ZoneRegistry zoneRegistry, + ConfigServerRestExecutor proxy) { + super(executor, accessLog); + this.zoneRegistry = zoneRegistry; + this.proxy = proxy; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: + return get(request); + case POST: + case PUT: + case DELETE: + case PATCH: + return proxy(request); + default: + return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + } + } catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse get(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/zone/v2")) { + return root(request); + } + return proxy(request); + } + + private HttpResponse proxy(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (!path.matches("/zone/v2/{environment}/{region}/{*}")) { + return notFound(path); + } + Environment environment = Environment.from(path.get("environment")); + RegionName region = RegionName.from(path.get("region")); + Optional<Zone> zone = zoneRegistry.getZone(environment, region); + if (!zone.isPresent()) { + throw new IllegalArgumentException("No such zone: " + environment.value() + "." + region.value()); + } + try { + return proxy.handle(new ProxyRequest(request, "/zone/v2/")); + } catch (ProxyException | IOException e) { + throw new RuntimeException(e); + } + } + + private HttpResponse root(HttpRequest request) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor uris = root.setArray("uris"); + zoneRegistry.zones().forEach(zone -> uris.addString(request.getUri() + .resolve("/zone/v2/") + .resolve(zone.environment().value() + "/") + .resolve(zone.region().value()) + .toString())); + Cursor zones = root.setArray("zones"); + zoneRegistry.zones().forEach(zone -> { + Cursor object = zones.addObject(); + object.setString("environment", zone.environment().value()); + object.setString("region", zone.region().value()); + }); + return new SlimeJsonResponse(slime); + } + + private HttpResponse notFound(Path path) { + return ErrorResponse.notFoundError("Nothing at " + path); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java new file mode 100644 index 00000000000..95dfed8b7b2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +package com.yahoo.vespa.hosted.controller.restapi.zone.v2; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index 6174a017a54..ae7223489c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import java.util.List; +import java.util.Collection; import java.util.Objects; +import java.util.Set; /** * Statistics about deployments on a platform version. This is immutable. @@ -16,20 +17,22 @@ import java.util.Objects; public class DeploymentStatistics { private final Version version; - private final ImmutableList<ApplicationId> failing; - private final ImmutableList<ApplicationId> production; + private final ImmutableSet<ApplicationId> failing; + private final ImmutableSet<ApplicationId> production; + private final ImmutableSet<ApplicationId> deploying; /** DO NOT USE. Public for serialization purposes */ - public DeploymentStatistics(Version version, List<ApplicationId> failingApplications, - List<ApplicationId> production) { + public DeploymentStatistics(Version version, Collection<ApplicationId> failingApplications, + Collection<ApplicationId> production, Collection<ApplicationId> deploying) { this.version = version; - this.failing = ImmutableList.copyOf(failingApplications); - this.production = ImmutableList.copyOf(production); + this.failing = ImmutableSet.copyOf(failingApplications); + this.production = ImmutableSet.copyOf(production); + this.deploying = ImmutableSet.copyOf(deploying); } /** Returns a statistics instance with the values as 0 */ public static DeploymentStatistics empty(Version version) { - return new DeploymentStatistics(version, ImmutableList.of(), ImmutableList.of()); + return new DeploymentStatistics(version, ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of()); } /** Returns the version these statistics are for */ @@ -39,23 +42,31 @@ public class DeploymentStatistics { * Returns the applications which have at least one job (of any type) which fails on this version, * excluding errors known to not be caused by this version */ - public List<ApplicationId> failing() { return failing; } + public Set<ApplicationId> failing() { return failing; } /** Returns the applications which have this version in production in at least one zone */ - public List<ApplicationId> production() { return production; } - + public Set<ApplicationId> production() { return production; } + + /** Returns the applications which are currently upgrading to this version */ + public Set<ApplicationId> deploying() { return deploying; } + /** Returns a version of this with the given failing application added */ public DeploymentStatistics withFailing(ApplicationId application) { - return new DeploymentStatistics(version, add(application, failing), production); + return new DeploymentStatistics(version, add(application, failing), production, deploying); } /** Returns a version of this with the given production application added */ public DeploymentStatistics withProduction(ApplicationId application) { - return new DeploymentStatistics(version, failing, add(application, production)); + return new DeploymentStatistics(version, failing, add(application, production), deploying); + } + + /** Returns a version of this with the given deploying application added */ + public DeploymentStatistics withDeploying(ApplicationId application) { + return new DeploymentStatistics(version, failing, production, add(application, deploying)); } - private ImmutableList<ApplicationId> add(ApplicationId application, ImmutableList<ApplicationId> list) { - ImmutableList.Builder<ApplicationId> b = new ImmutableList.Builder<>(); + private ImmutableSet<ApplicationId> add(ApplicationId application, ImmutableSet<ApplicationId> list) { + ImmutableSet.Builder<ApplicationId> b = new ImmutableSet.Builder<>(); b.addAll(list); b.add(application); return b.build(); @@ -75,4 +86,5 @@ public class DeploymentStatistics { public int hashCode() { return Objects.hash(version, failing, production); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 73e4eb4d527..d152cf80472 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.JobList; import java.net.URI; import java.time.Instant; @@ -28,6 +28,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity; + /** * Information about the current platform versions in use. * The versions in use are the set of all versions running in current applications, versions @@ -92,7 +94,8 @@ public class VersionStatus { Version systemVersion = infrastructureVersions.stream().sorted().findFirst().get(); Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, - controller.applications().asList()); + controller.applications().asList(), + controller.applications().deploymentTrigger().jobTimeoutLimit()); List<VespaVersion> versions = new ArrayList<>(); for (DeploymentStatistics statistics : deploymentStatistics) { @@ -126,7 +129,8 @@ public class VersionStatus { } private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions, - List<Application> applications) { + List<Application> applications, + Instant jobTimeoutLimit) { Map<Version, DeploymentStatistics> versionMap = new HashMap<>(); for (Version infrastructureVersion : infrastructureVersions) { @@ -142,41 +146,38 @@ public class VersionStatus { versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty); } - // List versions which have failing jobs, and versions which are in production + // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs // Failing versions - Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream() - .filter(jobStatus -> jobStatus.lastCompleted().isPresent()) - .filter(jobStatus -> jobStatus.lastCompleted().get().upgrade()) - .filter(jobStatus -> jobStatus.jobError().isPresent()) - .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version())); - for (Version v : failingJobsByVersion.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withFailing(application.id())); - } + JobList.from(application) + .failing() + .not().failingApplicationChange() + .not().failingBecause(outOfCapacity) + .mapToList(job -> job.lastCompleted().get().version()) + .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withFailing(application.id()))); // Succeeding versions - Map<Version, List<JobStatus>> succeedingJobsByVersions = jobs.jobStatus().values().stream() - .filter(jobStatus -> jobStatus.lastSuccess().isPresent()) - .filter(jobStatus -> jobStatus.type().isProduction()) - .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastSuccess().get().version())); - for (Version v : succeedingJobsByVersions.keySet()) { - versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withProduction(application.id())); - } + JobList.from(application) + .lastSuccess().present() + .production() + .mapToList(job -> job.lastSuccess().get().version()) + .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withProduction(application.id()))); + + // Deploying versions + JobList.from(application) + .upgrading() + .mapToList(job -> job.lastTriggered().get().version()) + .forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withDeploying(application.id()))); } return versionMap.values(); } - private static DeploymentStatistics emptyIfMissing(Version version, DeploymentStatistics statistics) { - return statistics == null ? DeploymentStatistics.empty(version) : statistics; - } - - private static VespaVersion createVersion(DeploymentStatistics statistics, + private static VespaVersion createVersion(DeploymentStatistics statistics, boolean isSystemVersion, Collection<String> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); - Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); + Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); // commitedAt ... VespaVersion.Confidence confidence; // Always compute confidence for system version if (isSystemVersion) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index de42a9fba4e..4bcee5782ee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -58,7 +58,7 @@ public class VespaVersion implements Comparable<VespaVersion> { return Confidence.broken; // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all - if (nonCanaryApplicationsBroken(failingOnThis, productionOnThis, releasedAt, controller.curator())) + if (nonCanaryApplicationsBroken(statistics.version(), failingOnThis, productionOnThis, releasedAt, controller.curator())) return Confidence.broken; // 'low' unless all canary applications are upgraded @@ -142,11 +142,12 @@ public class VespaVersion implements Comparable<VespaVersion> { } - private static boolean nonCanaryApplicationsBroken(ApplicationList failingOnThis, + private static boolean nonCanaryApplicationsBroken(Version version, + ApplicationList failingOnThis, ApplicationList productionOnThis, Instant releasedAt, CuratorDb curator) { - ApplicationList failingNonCanaries = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt); + ApplicationList failingNonCanaries = failingOnThis.without(UpgradePolicy.canary).startedFailingOnVersionAfter(version, releasedAt); ApplicationList productionNonCanaries = productionOnThis.without(UpgradePolicy.canary); if (productionNonCanaries.size() + failingNonCanaries.size() == 0 || curator.readIgnoreConfidence()) return false; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java index 9eef1dac70b..363a2ea19cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java @@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.rotation; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; -import com.yahoo.metrics.simple.Gauge; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.ApplicationAlias; -import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.jetbrains.annotations.NotNull; @@ -31,17 +30,16 @@ import java.util.stream.Collectors; public class ControllerRotationRepository implements RotationRepository { private static final Logger log = Logger.getLogger(ControllerRotationRepository.class.getName()); - - private static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations"; - private final Gauge remainingRotations; + public static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations"; private final ControllerDb controllerDb; private final Map<RotationId, Rotation> rotationsMap; + private final Metric metric; - public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, MetricReceiver metricReceiver) { + public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, Metric metric) { this.controllerDb = controllerDb; this.rotationsMap = buildRotationsMap(rotationConfig); - this.remainingRotations = metricReceiver.declareGauge(REMAINING_ROTATIONS_METRIC_NAME); + this.metric = metric; } private static Map<RotationId, Rotation> buildRotationsMap(RotationsConfig rotationConfig) { @@ -73,7 +71,7 @@ public class ControllerRotationRepository implements RotationRepository { .collect(Collectors.toSet()); } - if( ! deploymentSpec.globalServiceId().isPresent()) { + if (!deploymentSpec.globalServiceId().isPresent()) { return Collections.emptySet(); } @@ -84,13 +82,12 @@ public class ControllerRotationRepository implements RotationRepository { if (productionZoneCount >= 2) { return assignRotation(applicationId); - } - else { + } else { throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); } } - private boolean isCorp(DeploymentSpec.DeclaredZone zone) { + private static boolean isCorp(DeploymentSpec.DeclaredZone zone) { return zone.region().isPresent() && zone.region().get().value().contains("corp"); } @@ -139,7 +136,8 @@ public class ControllerRotationRepository implements RotationRepository { try { int freeRotationsCount = availableRotations().size(); log.log(LogLevel.INFO, "Rotation: {0} global rotations remaining", freeRotationsCount); - remainingRotations.sample(freeRotationsCount); + metric.set(REMAINING_ROTATIONS_METRIC_NAME, freeRotationsCount, + metric.createContext(Collections.emptyMap())); } catch (Exception e) { log.log(LogLevel.INFO, "Failed to report rotations metric", e); } diff --git a/controller-server/src/main/resources/configdefinitions/athenz.def b/controller-server/src/main/resources/configdefinitions/athenz.def index 4e27e3ebd07..6d10f3dee28 100644 --- a/controller-server/src/main/resources/configdefinitions/athenz.def +++ b/controller-server/src/main/resources/configdefinitions/athenz.def @@ -13,6 +13,10 @@ ztsUrl string # Athenz domain for controller identity. The domain is also used for Athenz tenancy integration. domain string +# Name of the internal user authentication passthru attribute +userAuthenticationPassThruAttribute string +# TODO Remove once migrated to Okta + # Athenz service name for controller identity service.name string diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerProxyMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerProxyMock.java new file mode 100644 index 00000000000..cc915d4d9a1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerProxyMock.java @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; +import com.yahoo.vespa.hosted.controller.proxy.ProxyException; +import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; + +import java.io.InputStream; +import java.util.Optional; +import java.util.Scanner; + +/** + * @author mpolden + */ +public class ConfigServerProxyMock extends AbstractComponent implements ConfigServerRestExecutor { + + private volatile ProxyRequest lastReceived = null; + private volatile String requestBody = null; + + @Override + public HttpResponse handle(ProxyRequest proxyRequest) throws ProxyException { + lastReceived = proxyRequest; + // Copy request body as the input stream is drained once the request completes + requestBody = asString(proxyRequest.getData()); + return new StringResponse("ok"); + } + + public Optional<ProxyRequest> lastReceived() { + return Optional.ofNullable(lastReceived); + } + + public Optional<String> lastRequestBody() { + return Optional.ofNullable(requestBody); + } + + private static String asString(InputStream inputStream) { + Scanner scanner = new Scanner(inputStream).useDelimiter("\\A"); + return scanner.hasNext() ? scanner.next() : ""; + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index aea66f3cd67..d0c1fd95427 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -30,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.athenz.NToken; @@ -80,6 +79,12 @@ public class ControllerTest { .region("corp-us-east-1") .build(); + private static final ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-west-1") + .build(); + @Test public void testDeployment() { // Setup system @@ -94,7 +99,7 @@ public class ControllerTest { // staging job - succeeding Version version1 = Version.fromString("6.1"); // Set in config server mock Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); assertFalse("Revision is currently not known", ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); tester.deployAndNotify(app1, applicationPackage, true, systemTest); @@ -104,12 +109,12 @@ public class ControllerTest { Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision(); assertTrue("Revision has been set during deployment", revision.isPresent()); assertStatus(JobStatus.initial(stagingTest) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) - .withCompletion(-1, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); // Causes first deployment job to be triggered assertStatus(JobStatus.initial(productionCorpUsEast1) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); tester.clock().advance(Duration.ofSeconds(1)); // production job (failing) @@ -117,10 +122,10 @@ public class ControllerTest { assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) // Triggered first without revision info - .withCompletion(-1, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()); // Re-triggering (due to failure) has revision info - + .withTriggering(version1, revision, false, "", tester.clock().instant()) // Triggered first without revision info + .withCompletion(42, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) + .withTriggering(version1, revision, false, "", tester.clock().instant()); // Re-triggering (due to failure) has revision info + assertStatus(expectedJobStatus, app1.id(), tester.controller()); // Simulate restart @@ -133,26 +138,29 @@ public class ControllerTest { InstanceName.from("default")))); assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); - tester.clock().advance(Duration.ofSeconds(1)); + + tester.clock().advance(Duration.ofHours(1)); + + tester.notifyJobCompletion(productionCorpUsEast1, app1, false); // Need to complete the job, or new jobs won't start. // system and staging test job - succeeding - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, false, systemTest); assertStatus(JobStatus.initial(systemTest) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) - .withCompletion(-1, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); // production job succeeding now tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1); expectedJobStatus = expectedJobStatus - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) - .withCompletion(-1, Optional.empty(), tester.clock().instant(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()); assertStatus(expectedJobStatus, app1.id(), tester.controller()); // causes triggering of next production job assertStatus(JobStatus.initial(productionUsEast3) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()), + .withTriggering(version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); @@ -163,7 +171,7 @@ public class ControllerTest { .environment(Environment.prod) .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); try { tester.deploy(systemTest, app1, applicationPackage); fail("Expected exception due to unallowed production deployment removal"); @@ -176,7 +184,7 @@ public class ControllerTest { JobStatus jobStatus = applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1); assertNotNull("Deployment job was not removed", jobStatus); assertEquals(42, jobStatus.lastCompleted().get().id()); - assertEquals("stagingTest completed successfully in build 42", jobStatus.lastCompleted().get().reason()); + assertEquals("staging-test completed", jobStatus.lastCompleted().get().reason()); // prod zone removal is allowed with override applicationPackage = new ApplicationPackageBuilder() @@ -205,13 +213,13 @@ public class ControllerTest { Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); // First deployment: An application change - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); - assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); + assertEquals("First deployment gets system version", systemVersion, app1.oldestDeployedVersion().get()); assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // Unexpected deployment @@ -228,19 +236,19 @@ public class ControllerTest { .region("us-west-1") .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); - assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); + assertEquals("Application change preserves version", systemVersion, app1.oldestDeployedVersion().get()); assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // A deployment to the new region gets the same version tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); - assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); + assertEquals("Application change preserves version", systemVersion, app1.oldestDeployedVersion().get()); assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); assertFalse("Change deployed", app1.deploying().isPresent()); @@ -253,7 +261,7 @@ public class ControllerTest { tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); - assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); + assertEquals("Version upgrade changes version", newSystemVersion, app1.oldestDeployedVersion().get()); assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion().get()); } @@ -322,13 +330,13 @@ public class ControllerTest { tester.notifyJobCompletion(component, app, true); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at initial failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); // Failure again -- failingSince should remain the same tester.clock().advance(Duration.ofMillis(1000)); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at second consecutive failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); // Success resets failingSince tester.clock().advance(Duration.ofMillis(1000)); @@ -346,13 +354,13 @@ public class ControllerTest { tester.notifyJobCompletion(component, app, true); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at initial failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); // Failure again -- failingSince should remain the same tester.clock().advance(Duration.ofMillis(1000)); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at second consecutive failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); } private Optional<JobStatus.JobRun> firstFailing(Application application, DeploymentTester tester) { @@ -425,7 +433,7 @@ public class ControllerTest { // app1: staging-test job fails with out of capacity and is added to the front of the queue tester.deploy(stagingTest, app1, applicationPackage); tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); - assertEquals(stagingTest.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(stagingTest.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(project1, buildSystem.jobs().get(0).projectId()); // app2 and app3: Completes deployment @@ -437,6 +445,7 @@ public class ControllerTest { // app1: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by // out of capacity retry mechanism tester.clock().advance(Duration.ofMinutes(15)); + tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); // Clear the previous staging test tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, false, systemTest); tester.deploy(stagingTest, app1, applicationPackage); @@ -444,12 +453,13 @@ public class ControllerTest { tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); assertTrue("No jobs queued", buildSystem.jobs().isEmpty()); - // app2 and app3: New change triggers staging-test jobs + // app2 and app3: New change triggers system-test jobs + // Provide a changed application package, too, or the deployment is a no-op. tester.notifyJobCompletion(component, app2, true); - tester.deployAndNotify(app2, applicationPackage, true, systemTest); + tester.deployAndNotify(app2, applicationPackage2, true, systemTest); tester.notifyJobCompletion(component, app3, true); - tester.deployAndNotify(app3, applicationPackage, true, systemTest); + tester.deployAndNotify(app3, applicationPackage2, true, systemTest); assertEquals(2, buildSystem.jobs().size()); @@ -457,19 +467,19 @@ public class ControllerTest { // back of the queue tester.clock().advance(Duration.ofHours(3)); tester.clock().advance(Duration.ofMinutes(50)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); List<BuildJob> nextJobs = buildSystem.takeJobsToRun(); assertEquals(2, nextJobs.size()); - assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(0).jobName()); assertEquals(project2, nextJobs.get(0).projectId()); - assertEquals(stagingTest.id(), nextJobs.get(1).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(1).jobName()); assertEquals(project3, nextJobs.get(1).projectId()); // And finally the requeued job for app1 nextJobs = buildSystem.takeJobsToRun(); assertEquals(1, nextJobs.size()); - assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(0).jobName()); assertEquals(project1, nextJobs.get(0).projectId()); } @@ -480,20 +490,6 @@ public class ControllerTest { assertEquals(expectedStatus, existingStatus); } - private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError) { - return new JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - jobError - ); - } - - private JobReport mockReport(Application application, JobType jobType, boolean success) { - return mockReport(application, jobType, JobError.from(success)); - } - @Test public void testGlobalRotations() throws IOException { // Setup tester and app def @@ -527,8 +523,7 @@ public class ControllerTest { TenantId tenant = tester.createTenant("tenant1", "domain1", 11L); Application app = tester.createApplication(tenant, "app1", "default", 1); - try (Lock lock = tester.controller().applications().lock(app.id())) { - LockedApplication application = tester.controller().applications().require(app.id(), lock); + tester.controller().applications().lockedOrThrow(app.id(), application -> { application = application.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.3")))); applications.store(application); try { @@ -537,7 +532,7 @@ public class ControllerTest { } catch (IllegalArgumentException e) { assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as version change to 6.3 is not tested", e.getMessage()); } - } + }); } @Test @@ -557,11 +552,7 @@ public class ControllerTest { // Load test data data ApplicationSerializer serializer = new ApplicationSerializer(); byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); - Slime slime = SlimeUtils.jsonToSlime(json); - Application application = serializer.fromSlime(slime); - try (Lock lock = tester.controller().applications().lock(application.id())) { - tester.controller().applications().store(new LockedApplication(application, lock)); - } + Application application = tester.controllerTester().createApplication(SlimeUtils.jsonToSlime(json)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .upgradePolicy("canary") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 4e3c15ea1a4..8f9c22f8b81 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -181,12 +181,9 @@ public final class ControllerTester { public Application createApplication(TenantId tenant, String applicationName, String instanceName, long projectId) { ApplicationId applicationId = applicationId(tenant.id(), applicationName, instanceName); controller().applications().createApplication(applicationId, Optional.of(TestIdentities.userNToken)); - try (Lock lock = controller().applications().lock(applicationId)) { - LockedApplication lockedApplication = controller().applications().require(applicationId, lock) - .withProjectId(projectId); - controller().applications().store(lockedApplication); - return lockedApplication; - } + controller().applications().lockedOrThrow(applicationId, lockedApplication -> + controller().applications().store(lockedApplication.withProjectId(projectId))); + return controller().applications().require(applicationId); } public void deploy(Application application, Zone zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index bf21467bc8d..18332942c24 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -10,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -19,15 +22,39 @@ import java.util.Optional; /** * @author mpolden */ -public class ZoneRegistryMock implements ZoneRegistry { +public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry { private final Map<Zone, Duration> deploymentTimeToLive = new HashMap<>(); + private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); + private List<Zone> zones = new ArrayList<>(); + private SystemName system = SystemName.main; + + @Inject + public ZoneRegistryMock() { + this.zones.add(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("corp-us-east-1"))); + this.zones.add(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("us-east-3"))); + this.zones.add(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("us-west-1"))); + } - public void setDeploymentTimeToLive(Zone zone, Duration duration) { + public ZoneRegistryMock setDeploymentTimeToLive(Zone zone, Duration duration) { deploymentTimeToLive.put(zone, duration); + return this; } - private SystemName system = SystemName.main; + public ZoneRegistryMock setDefaultRegionForEnvironment(Environment environment, RegionName region) { + defaultRegionForEnvironment.put(environment, region); + return this; + } + + public ZoneRegistryMock setZones(List<Zone> zones) { + this.zones = zones; + return this; + } + + public ZoneRegistryMock setSystem(SystemName system) { + this.system = system; + return this; + } @Override public SystemName system() { @@ -36,12 +63,13 @@ public class ZoneRegistryMock implements ZoneRegistry { @Override public List<Zone> zones() { - return Collections.singletonList(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("corp-us-east-1"))); + return Collections.unmodifiableList(zones); } @Override public Optional<Zone> getZone(Environment environment, RegionName region) { - return zones().stream().filter(z -> z.environment().equals(environment) && z.region().equals(region)).findFirst(); + return zones().stream().filter(z -> z.environment().equals(environment) && + z.region().equals(region)).findFirst(); } @Override @@ -64,6 +92,11 @@ public class ZoneRegistryMock implements ZoneRegistry { } @Override + public Optional<RegionName> getDefaultRegion(Environment environment) { + return Optional.ofNullable(defaultRegionForEnvironment.get(environment)); + } + + @Override public URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application) { return URI.create("http://monitoring-system.test/?environment=" + environment.value() + "®ion=" + name.value() + "&application=" + application.toShortString()); @@ -74,7 +107,4 @@ public class ZoneRegistryMock implements ZoneRegistry { return URI.create("http://dashboard.test"); } - public void setSystem(SystemName system) { - this.system = system; - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 79fd717a24f..2b0e953c12c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -16,17 +16,19 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.maintenance.FailureRedeployer; +import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.unknown; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -44,7 +46,7 @@ public class DeploymentTester { private final ControllerTester tester; private final Upgrader upgrader; - private final FailureRedeployer failureRedeployer; + private final ReadyJobsTrigger readyJobTrigger; public DeploymentTester() { this(new ControllerTester()); @@ -55,18 +57,20 @@ public class DeploymentTester { tester.curator().writeUpgradesPerMinute(100); this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()), tester.curator()); - this.failureRedeployer = new FailureRedeployer(tester.controller(), maintenanceInterval, - new JobControl(tester.curator())); + this.readyJobTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, + new JobControl(tester.curator())); } public Upgrader upgrader() { return upgrader; } - public FailureRedeployer failureRedeployer() { return failureRedeployer; } + public ReadyJobsTrigger readyJobTrigger() { return readyJobTrigger; } public Controller controller() { return tester.controller(); } public ApplicationController applications() { return tester.controller().applications(); } + // TODO: This thing simulates the wrong thing: the build system won't hold the jobs that are running, + // and so these should be consumed immediately upon triggering, and be "somewhere else" while running. public BuildSystem buildSystem() { return tester.controller().applications().deploymentTrigger().buildSystem(); } public DeploymentTrigger deploymentTrigger() { return tester.controller().applications().deploymentTrigger(); } @@ -115,8 +119,13 @@ public class DeploymentTester { /** Simulate the full lifecycle of an application deployment as declared in given application package */ public Application createAndDeploy(String applicationName, int projectId, ApplicationPackage applicationPackage) { - tester.createTenant("tenant1", "domain1", 1L); - Application application = tester.createApplication(new TenantId("tenant1"), applicationName, "default", projectId); + TenantId tenantId = tester.createTenant("tenant1", "domain1", 1L); + return createAndDeploy(tenantId, applicationName, projectId, applicationPackage); + } + + /** Simulate the full lifecycle of an application deployment as declared in given application package */ + public Application createAndDeploy(TenantId tenantId, String applicationName, int projectId, ApplicationPackage applicationPackage) { + Application application = tester.createApplication(tenantId, applicationName, "default", projectId); deployCompletely(application, applicationPackage); return applications().require(application.id()); } @@ -126,6 +135,11 @@ public class DeploymentTester { return createAndDeploy(applicationName, projectId, applicationPackage(upgradePolicy)); } + /** Simulate the full lifecycle of an application deployment to prod.us-west-1 with the given upgrade policy */ + public Application createAndDeploy(TenantId tenantId, String applicationName, int projectId, String upgradePolicy) { + return createAndDeploy(tenantId, applicationName, projectId, applicationPackage(upgradePolicy)); + } + /** Complete an ongoing deployment */ public void deployCompletely(String applicationName) { deployCompletely(applications().require(ApplicationId.from("tenant1", applicationName, "default")), @@ -139,6 +153,20 @@ public class DeploymentTester { completeDeployment(application, applicationPackage, Optional.empty(), true); } + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, boolean success) { + return jobReport(application, jobType, Optional.ofNullable(success ? null : unknown)); + } + + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { + return new DeploymentJobs.JobReport( + application.id(), + jobType, + application.deploymentJobs().projectId().get(), + 42, + jobError + ); + } + /** Deploy application using the given application package, but expecting to stop after test phases */ public void deployTestOnly(Application application, ApplicationPackage applicationPackage) { notifyJobCompletion(JobType.component, application, true); @@ -154,7 +182,7 @@ public class DeploymentTester { jobs = jobs.stream().filter(job -> ! job.isProduction()).collect(Collectors.toList()); for (JobType job : jobs) { boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false); - deployAndNotify(application, applicationPackage, !failJob, false, job); + deployAndNotify(application, applicationPackage, ! failJob, false, job); if (failJob) { break; } @@ -171,10 +199,11 @@ public class DeploymentTester { } public void notifyJobCompletion(JobType jobType, Application application, boolean success) { - notifyJobCompletion(jobType, application, DeploymentJobs.JobError.from(success)); + notifyJobCompletion(jobType, application, Optional.ofNullable(success ? null : unknown)); } public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError) { + clock().advance(Duration.ofMillis(1)); applications().notifyJobCompletion(jobReport(application, jobType, jobError)); } @@ -211,7 +240,7 @@ public class DeploymentTester { deployAndNotify(application, applicationPackage, success, true, jobs); } - public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, + public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, boolean expectOnlyTheseJobs, JobType... jobs) { consumeJobs(application, expectOnlyTheseJobs, jobs); for (JobType job : jobs) { @@ -225,21 +254,20 @@ public class DeploymentTester { /** Assert that the sceduled jobs of this application are exactly those given, and take them */ private void consumeJobs(Application application, boolean expectOnlyTheseJobs, JobType... jobs) { for (JobType job : jobs) { - Optional<BuildService.BuildJob> buildJob = findJob(application, job); - assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent()); - assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId()); - assertEquals(job.id(), buildJob.get().jobName()); + BuildService.BuildJob buildJob = findJob(application, job); + assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.projectId()); + assertEquals(job.jobName(), buildJob.jobName()); } if (expectOnlyTheseJobs) assertEquals(jobs.length, countJobsOf(application)); buildSystem().removeJobs(application.id()); } - private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) { + private BuildService.BuildJob findJob(Application application, JobType jobType) { for (BuildService.BuildJob job : buildSystem().jobs()) - if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id())) - return Optional.of(job); - return Optional.empty(); + if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.jobName())) + return job; + throw new NoSuchElementException(jobType + " is not scheduled for " + application); } private int countJobsOf(Application application) { @@ -247,17 +275,8 @@ public class DeploymentTester { .filter(job -> job.projectId() == application.deploymentJobs().projectId().get()) .count(); } - private DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { - return new DeploymentJobs.JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - jobError - ); - } - private static ApplicationPackage applicationPackage(String upgradePolicy) { + public static ApplicationPackage applicationPackage(String upgradePolicy) { return new ApplicationPackageBuilder() .upgradePolicy(upgradePolicy) .environment(Environment.prod) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 022fa705def..10f8e80f318 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.maintenance.BlockedChangeDeployer; +import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import org.junit.Test; @@ -59,10 +59,11 @@ public class DeploymentTriggerTest { // system-test fails and is retried tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest); assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size()); - tester.buildSystem().takeJobsToRun(); - assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); - tester.clock().advance(Duration.ofHours(4).plus(Duration.ofSeconds(1))); - tester.failureRedeployer().maintain(); // Causes retry of systemTests + tester.clock().advance(Duration.ofHours(1)); + tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest); + tester.clock().advance(Duration.ofHours(1)); + assertEquals("Nothing scheduled", 0, tester.buildSystem().jobs().size()); + tester.readyJobTrigger().maintain(); // Causes retry of systemTests assertEquals("Scheduled retry", 1, tester.buildSystem().jobs().size()); tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); @@ -70,9 +71,9 @@ public class DeploymentTriggerTest { // staging-test times out and is retried tester.buildSystem().takeJobsToRun(); tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertEquals("Retried dead job", 1, tester.buildSystem().jobs().size()); - assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.stagingTest.jobName(), tester.buildSystem().jobs().get(0).jobName()); } @Test @@ -127,16 +128,16 @@ public class DeploymentTriggerTest { // 30 seconds pass, us-west-1 is triggered tester.clock().advance(Duration.ofSeconds(30)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); // Consume us-west-1 job without reporting completion assertEquals(1, buildSystem.jobs().size()); - assertEquals(JobType.productionUsWest1.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.jobName(), buildSystem.jobs().get(0).jobName()); buildSystem.takeJobsToRun(); // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress tester.clock().advance(Duration.ofMinutes(3)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); // us-west-1 completes @@ -144,18 +145,18 @@ public class DeploymentTriggerTest { tester.notifyJobCompletion(JobType.productionUsWest1, application, true); // Delayed trigger does nothing as not enough time has passed after us-west-1 completion - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); // 3 minutes pass, us-central-1 is triggered tester.clock().advance(Duration.ofMinutes(3)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); // Delayed trigger job runs again, with nothing to trigger tester.clock().advance(Duration.ofMinutes(10)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } @@ -184,8 +185,8 @@ public class DeploymentTriggerTest { // Deploys in two regions in parallel assertEquals(2, tester.buildSystem().jobs().size()); - assertEquals(JobType.productionUsEast3.id(), tester.buildSystem().jobs().get(0).jobName()); - assertEquals(JobType.productionUsWest1.id(), tester.buildSystem().jobs().get(1).jobName()); + assertEquals(JobType.productionUsEast3.jobName(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.jobName(), tester.buildSystem().jobs().get(1).jobName()); tester.buildSystem().takeJobsToRun(); tester.deploy(JobType.productionUsWest1, application, applicationPackage, false); @@ -269,9 +270,9 @@ public class DeploymentTriggerTest { public void testBlockRevisionChange() { ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:30:00.00Z")); // Tuesday, 17:30 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); @@ -290,7 +291,7 @@ public class DeploymentTriggerTest { tester.clock().advance(Duration.ofHours(1)); // --------------- Enter block window: 18:30 - blockedChangeDeployer.run(); + readyJobsTrigger.run(); assertEquals(0, tester.buildSystem().jobs().size()); String searchDefinition = @@ -304,7 +305,7 @@ public class DeploymentTriggerTest { tester.deployTestOnly(app, changedApplication); - blockedChangeDeployer.run(); + readyJobsTrigger.run(); assertEquals(0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(2)); // ---------------- Exit block window: 20:30 @@ -317,14 +318,14 @@ public class DeploymentTriggerTest { @Test public void testUpgradingButNoJobStarted() { DeploymentTester tester = new DeploymentTester(); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); LockedApplication app = (LockedApplication)tester.createAndDeploy("default0", 3, "default"); // Store that we are upgrading but don't start the system-tests job tester.controller().applications().store(app.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.2"))))); assertEquals(0, tester.buildSystem().jobs().size()); - blockedChangeDeployer.run(); + readyJobsTrigger.run(); assertEquals(1, tester.buildSystem().jobs().size()); assertEquals("system-test", tester.buildSystem().jobs().get(0).jobName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java index 0293ea08d65..1b1a4feaa4e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java @@ -13,6 +13,7 @@ import java.time.Duration; import java.util.EnumMap; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.QUEUED; @@ -161,11 +162,11 @@ public class MockBuildService implements BuildService { jobType, projectId, 42, - JobError.from(success) + Optional.ofNullable(success ? null : JobError.unknown) )); } - private BuildJob buildJob() { return new BuildJob(projectId, jobType.id()); } + private BuildJob buildJob() { return new BuildJob(projectId, jobType.jobName()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java new file mode 100644 index 00000000000..f5be882fcb8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -0,0 +1,121 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.Optional; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author jvenstad + */ +public class ApplicationOwnershipConfirmerTest { + + private MockOwnershipIssues issues; + private ApplicationOwnershipConfirmer confirmer; + private DeploymentTester tester; + + @Before + public void setup() { + tester = new DeploymentTester(); + issues = new MockOwnershipIssues(); + confirmer = new ApplicationOwnershipConfirmer(tester.controller(), Duration.ofDays(1), new JobControl(new MockCuratorDb()), issues); + } + + @Test + public void testConfirmation() { + TenantId property = tester.controllerTester().createTenant("property", "domain", 1L); + tester.createAndDeploy(property, "application", 1, "default"); + Supplier<Application> propertyApp = () -> tester.controller().applications().require(ApplicationId.from("property", "application", "default")); + + TenantId user = new TenantId("by-user"); + tester.controller().tenants().addTenant(Tenant.createUserTenant(user), Optional.empty()); + tester.createAndDeploy(user, "application", 2, "default"); + Supplier<Application> userApp = () -> tester.controller().applications().require(ApplicationId.from("by-user", "application", "default")); + + assertFalse("No issue is initially stored for a new application.", propertyApp.get().ownershipIssueId().isPresent()); + assertFalse("No issue is initially stored for a new application.", userApp.get().ownershipIssueId().isPresent()); + assertFalse("No escalation has been attempted for a new application", issues.escalatedForProperty || issues.escalatedForUser); + + // Set response from the issue mock, which will be obtained by the maintainer on issue filing. + Optional<IssueId> issueId = Optional.of(IssueId.from("1")); + issues.response = issueId; + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("Confirmation issue has been filed for property owned application.", issueId, propertyApp.get().ownershipIssueId()); + assertEquals("Confirmation issue has been filed for user owned application.", issueId, userApp.get().ownershipIssueId()); + assertTrue("Both applications have had their responses ensured.", issues.escalatedForProperty && issues.escalatedForUser); + + // No new issue is created, so return empty now. + issues.response = Optional.empty(); + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, propertyApp.get().ownershipIssueId()); + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, userApp.get().ownershipIssueId()); + + // The user deletes its production deployment — see that the issue is forgotten. + assertEquals("Confirmation issue for user is sitll open.", issueId, userApp.get().ownershipIssueId()); + tester.controller().applications().deactivate(userApp.get(), userApp.get().productionDeployments().keySet().stream().findAny().get()); + assertTrue("No production deployments are listed for user.", userApp.get().productionDeployments().isEmpty()); + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("Confirmation issue has been forgotten for application without production deployments.", Optional.empty(), userApp.get().ownershipIssueId()); + + // Time has passed, and a new confirmation issue is in order for the property which is still in production. + Optional<IssueId> issueId2 = Optional.of(IssueId.from("2")); + issues.response = issueId2; + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, propertyApp.get().ownershipIssueId()); + assertEquals("Confirmation issue for application without production deployments has not been filed.", Optional.empty(), userApp.get().ownershipIssueId()); + + } + + private class MockOwnershipIssues implements OwnershipIssues { + + private Optional<IssueId> response; + private boolean escalatedForProperty = false; + private boolean escalatedForUser = false; + + @Override + public Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, PropertyId propertyId) { + return response; + } + + @Override + public Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, User owner) { + return response; + } + + @Override + public void ensureResponse(IssueId issueId, Optional<PropertyId> propertyId) { + if (propertyId.isPresent()) escalatedForProperty = true; + else escalatedForUser = true; + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index b5ea8e0a36f..e57edcf6da0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -54,7 +54,7 @@ public class DeploymentIssueReporterTest { public void setup() { tester = new DeploymentTester(); issues = new MockDeploymentIssues(); - reporter = new DeploymentIssueReporter(tester.controller(), issues, Duration.ofMinutes(5), new JobControl(new MockCuratorDb())); + reporter = new DeploymentIssueReporter(tester.controller(), issues, Duration.ofDays(1), new JobControl(new MockCuratorDb())); } @Test @@ -135,9 +135,7 @@ public class DeploymentIssueReporterTest { // app3 now has a new failure past max failure age; see that a new issue is filed. tester.notifyJobCompletion(component, app3, true); - tester.deployAndNotify(app3, applicationPackage, true, systemTest); - tester.deployAndNotify(app3, applicationPackage, true, stagingTest); - tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1); + tester.deployAndNotify(app3, applicationPackage, false, systemTest); tester.clock().advance(maxInactivity.plus(maxFailureAge)); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index d3907e27456..148d11e8b38 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -5,14 +5,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Assert; import org.junit.Test; import java.time.Duration; +import static org.junit.Assert.assertEquals; + /** * @author smorgrav */ @@ -23,20 +25,24 @@ public class DeploymentMetricsMaintainerTest { ControllerTester tester = new ControllerTester(); ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); - // Pre condition: no metric info on the deployment + // Pre condition: no metric info on neither application nor deployment + assertEquals(0, tester.controller().applications().require(app).metrics().queryServiceQuality(), 0); Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); - Assert.assertEquals(0, deployment.metrics().documentCount(), Double.MIN_VALUE); + assertEquals(0, deployment.metrics().documentCount(), 0); DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofMinutes(10), new JobControl(new MockCuratorDb())); maintainer.maintain(); // Post condition: - deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); - Assert.assertEquals(1, deployment.metrics().queriesPerSecond(), Double.MIN_VALUE); - Assert.assertEquals(2, deployment.metrics().writesPerSecond(), Double.MIN_VALUE); - Assert.assertEquals(3, deployment.metrics().documentCount(), Double.MIN_VALUE); - Assert.assertEquals(4, deployment.metrics().queryLatencyMillis(), Double.MIN_VALUE); - Assert.assertEquals(5, deployment.metrics().writeLatencyMillis(), Double.MIN_VALUE); + Application application = tester.controller().applications().require(app); + assertEquals(0.5, application.metrics().queryServiceQuality(), Double.MIN_VALUE); + assertEquals(0.7, application.metrics().writeServiceQuality(), Double.MIN_VALUE); + deployment = application.deployments().values().stream().findAny().get(); + assertEquals(1, deployment.metrics().queriesPerSecond(), Double.MIN_VALUE); + assertEquals(2, deployment.metrics().writesPerSecond(), Double.MIN_VALUE); + assertEquals(3, deployment.metrics().documentCount(), Double.MIN_VALUE); + assertEquals(4, deployment.metrics().queryLatencyMillis(), Double.MIN_VALUE); + assertEquals(5, deployment.metrics().writeLatencyMillis(), Double.MIN_VALUE); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 2782dd6ec3b..fd00123c697 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -62,15 +62,16 @@ public class FailureRedeployerTest { // Another version is released, which cancels any pending upgrades to lower versions version = Version.fromString("5.2"); tester.updateVersionStatus(version); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); // Finish previous production job. tester.upgrader().maintain(); assertEquals("Application starts upgrading to new version", 1, tester.buildSystem().jobs().size()); assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); // Failure redeployer does not retry failing job for prod.us-east-3 as there's an ongoing deployment tester.clock().advance(Duration.ofMinutes(1)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertFalse("Job is not retried", tester.buildSystem().jobs().stream() - .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id()))); + .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.jobName()))); // Test environments pass tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); @@ -86,7 +87,7 @@ public class FailureRedeployerTest { // Failure redeployer retries job tester.clock().advance(Duration.ofMinutes(5)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertEquals("Job is retried", 1, tester.buildSystem().jobs().size()); // Production job finally succeeds @@ -109,14 +110,14 @@ public class FailureRedeployerTest { tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); // staging-test starts, but does not complete - assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); - tester.failureRedeployer().maintain(); + assertEquals(DeploymentJobs.JobType.stagingTest.jobName(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + tester.readyJobTrigger().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); // Just over 12 hours pass, job is retried tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); - tester.failureRedeployer().maintain(); - assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + tester.readyJobTrigger().maintain(); + assertEquals(DeploymentJobs.JobType.stagingTest.jobName(), tester.buildSystem().takeJobsToRun().get(0).jobName()); // Deployment completes tester.deploy(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); @@ -168,7 +169,7 @@ public class FailureRedeployerTest { // Failure re-deployer does not retry failing system-test job as it failed for an older change tester.clock().advance(Duration.ofMinutes(5)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); } @@ -212,11 +213,11 @@ public class FailureRedeployerTest { // Production job starts, but does not complete assertEquals(1, tester.buildSystem().jobs().size()); - assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.jobName(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); // Failure re-deployer runs - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); // Deployment completes @@ -241,7 +242,7 @@ public class FailureRedeployerTest { Application application = tester.controllerTester().createApplication(slime); // Failure redeployer does not restart deployment - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); } @@ -261,7 +262,7 @@ public class FailureRedeployerTest { tester.controllerTester().createApplication(slime); // Failure redeployer does not restart deployment - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 621e189ba37..a7458f9f8ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -71,7 +71,7 @@ public class MetricsReporterTest { metricsReporter.maintain(); assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); - // Deploy 3 apps successfully + // Deploy all apps successfully Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); Application app2 = tester.createApplication("app2", "tenant1", 2, 22L); Application app3 = tester.createApplication("app3", "tenant1", 3, 33L); @@ -79,6 +79,7 @@ public class MetricsReporterTest { tester.deployCompletely(app1, applicationPackage); tester.deployCompletely(app2, applicationPackage); tester.deployCompletely(app3, applicationPackage); + tester.deployCompletely(app4, applicationPackage); metricsReporter.maintain(); assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 4886eba40b6..13636122cfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -49,7 +49,7 @@ public class OutstandingChangeDeployerTest { List<BuildService.BuildJob> jobs = tester.buildSystem().jobs(); assertEquals(1, jobs.size()); assertEquals(11, jobs.get(0).projectId()); - assertEquals(DeploymentJobs.JobType.systemTest.id(), jobs.get(0).jobName()); + assertEquals(DeploymentJobs.JobType.systemTest.jobName(), jobs.get(0).jobName()); assertFalse(tester.application("app1").hasOutstandingChange()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 0414cda3f55..8839f6a5a18 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -51,7 +51,7 @@ public class UpgraderTest { tester.upgrader().maintain(); assertEquals("All already on the right version: Nothing to do", 0, tester.buildSystem().jobs().size()); - // --- A new version is released - everything goes smoothly + // --- 5.1 is released - everything goes smoothly version = Version.fromString("5.1"); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -86,7 +86,7 @@ public class UpgraderTest { tester.upgrader().maintain(); assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size()); - // --- A new version is released - which fails a Canary + // --- 5.2 is released - which fails a Canary version = Version.fromString("5.2"); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -95,12 +95,23 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgradeWithError(canary0, version, "canary", DeploymentJobs.JobType.stagingTest); assertEquals("Other Canary was cancelled", 2, tester.buildSystem().jobs().size()); + // TODO: Cancelled would mean it was triggerd, removed from the build system, but never reported in. + // Thus, the expected number of jobs should be 1, above: the retrying canary0. + // Further, canary1 should be retried after the timeout period of 12 hours, but verifying this is + // not possible when jobs are consumed form the build system on notification, rather than on deploy. tester.updateVersionStatus(version); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); assertEquals("Version broken, but Canaries should keep trying", 2, tester.buildSystem().jobs().size()); + // Exhaust canary retries. + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, canary1, false); + tester.clock().advance(Duration.ofHours(1)); + tester.deployAndNotify(canary0, DeploymentTester.applicationPackage("canary"), false, DeploymentJobs.JobType.stagingTest); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, canary1, false); + //tester.deployAndNotify(canary1, DeploymentTester.applicationPackage("canary"), false, DeploymentJobs.JobType.stagingTest); + // --- A new version is released - which repairs the Canary app and fails a default version = Version.fromString("5.3"); tester.updateVersionStatus(version); @@ -128,11 +139,15 @@ public class UpgraderTest { tester.completeUpgrade(default2, version, "default"); tester.updateVersionStatus(version); - assertEquals("Not enough evidence to mark this neither broken nor high", + assertEquals("Not enough evidence to mark this as neither broken nor high", VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); - tester.upgrader().maintain(); + assertEquals("Upgrade with error should retry", 1, tester.buildSystem().jobs().size()); + // Finish previous run, with exhausted retry. + tester.clock().advance(Duration.ofHours(1)); + tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, default0, false); + // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold // Deploy application change tester.deployCompletely("default0"); @@ -148,51 +163,59 @@ public class UpgraderTest { assertEquals("Applications are on 5.3 - nothing to do", 0, tester.buildSystem().jobs().size()); // --- Starting upgrading to a new version which breaks, causing upgrades to commence on the previous version - version = Version.fromString("5.4"); + Version version54 = Version.fromString("5.4"); Application default3 = tester.createAndDeploy("default3", 5, "default"); // need 4 to break a version Application default4 = tester.createAndDeploy("default4", 5, "default"); - tester.updateVersionStatus(version); + tester.updateVersionStatus(version54); tester.upgrader().maintain(); // cause canary upgrades to 5.4 - tester.completeUpgrade(canary0, version, "canary"); - tester.completeUpgrade(canary1, version, "canary"); - tester.updateVersionStatus(version); + tester.completeUpgrade(canary0, version54, "canary"); + tester.completeUpgrade(canary1, version54, "canary"); + tester.updateVersionStatus(version54); assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size()); - assertEquals(version, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); - tester.completeUpgrade(default0, version, "default"); + assertEquals(version54, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + tester.completeUpgrade(default0, version54, "default"); // State: Default applications started upgrading to 5.4 (and one completed) - version = Version.fromString("5.5"); - tester.updateVersionStatus(version); + Version version55 = Version.fromString("5.5"); + tester.updateVersionStatus(version55); tester.upgrader().maintain(); // cause canary upgrades to 5.5 - tester.completeUpgrade(canary0, version, "canary"); - tester.completeUpgrade(canary1, version, "canary"); - tester.updateVersionStatus(version); + tester.completeUpgrade(canary0, version55, "canary"); + tester.completeUpgrade(canary1, version55, "canary"); + tester.updateVersionStatus(version55); assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size()); - assertEquals(version, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + assertEquals(version55, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + tester.completeUpgrade(default1, version54, "default"); + tester.completeUpgrade(default2, version54, "default"); + tester.completeUpgradeWithError(default3, version54, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default4, version54, "default", DeploymentJobs.JobType.productionUsWest1); // State: Default applications started upgrading to 5.5 - tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.stagingTest); - tester.completeUpgradeWithError(default1, version, "default", DeploymentJobs.JobType.stagingTest); - tester.completeUpgradeWithError(default2, version, "default", DeploymentJobs.JobType.stagingTest); - tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.productionUsWest1); - tester.completeUpgrade(default4, version, "default"); - tester.updateVersionStatus(version); + tester.upgrader().maintain(); + tester.completeUpgradeWithError(default0, version55, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default1, version55, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default2, version55, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default3, version55, "default", DeploymentJobs.JobType.productionUsWest1); + tester.updateVersionStatus(version55); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + + // Finish running job, without retry. + tester.clock().advance(Duration.ofHours(1)); + tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsWest1, default3, false); + tester.upgrader().maintain(); - assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken", - 3, tester.buildSystem().jobs().size()); - assertEquals("5.4", ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version().toString()); - assertEquals("5.4", ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version().toString()); + assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken: " + + "This is default3 since it failed upgrade on both 5.4 and 5.5", + 1, tester.buildSystem().jobs().size()); assertEquals("5.4", ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version().toString()); } @@ -451,9 +474,9 @@ public class UpgraderTest { public void testBlockVersionChangeHalfwayThough() { ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); @@ -483,12 +506,12 @@ public class UpgraderTest { // One hour passes, time is 19:00, still no upgrade tester.clock().advance(Duration.ofHours(1)); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); // Another hour pass, time is 20:00 and application upgrades tester.clock().advance(Duration.ofHours(1)); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1); tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); @@ -505,9 +528,9 @@ public class UpgraderTest { public void testBlockVersionChangeHalfwayThoughThenNewVersion() { ManualClock clock = new ManualClock(Instant.parse("2017-09-29T16:00:00.00Z")); // Friday, 16:00 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); @@ -542,14 +565,14 @@ public class UpgraderTest { version = Version.fromString("5.2"); tester.updateVersionStatus(version); tester.upgrader().maintain(); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); assertTrue("Nothing is scheduled", tester.buildSystem().jobs().isEmpty()); // Monday morning: We are not blocked tester.clock().advance(Duration.ofDays(1)); // Sunday, 17:00 tester.clock().advance(Duration.ofHours(17)); // Monday, 10:00 tester.upgrader().maintain(); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); // We proceed with the new version in the expected order, not starting with the previously blocked version: // Test jobs are run with the new version, but not production as we are in the block window tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); @@ -584,7 +607,7 @@ public class UpgraderTest { Application default3 = tester.createAndDeploy("default3", 6, "default"); Application default4 = tester.createAndDeploy("default4", 7, "default"); - assertEquals(version, default0.deployedVersion().get()); + assertEquals(version, default0.oldestDeployedVersion().get()); // New version is released version = Version.fromString("5.1"); @@ -610,8 +633,16 @@ public class UpgraderTest { tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.systemTest); tester.updateVersionStatus(version); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + // Exhaust retries and finish runs + tester.clock().advance(Duration.ofHours(1)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default0, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default1, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default2, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default3, false); + // 5th app never reports back and has a dead job, but no ongoing change Application deadLocked = tester.applications().require(default4.id()); assertTrue("Jobs in progress", deadLocked.deploymentJobs().isRunning(tester.controller().applications().deploymentTrigger().jobTimeoutLimit())); @@ -633,20 +664,10 @@ public class UpgraderTest { tester.completeUpgrade(default2, version, "default"); tester.completeUpgrade(default3, version, "default"); - assertEquals(version, tester.application(default0.id()).deployedVersion().get()); - assertEquals(version, tester.application(default1.id()).deployedVersion().get()); - assertEquals(version, tester.application(default2.id()).deployedVersion().get()); - assertEquals(version, tester.application(default3.id()).deployedVersion().get()); - - // Over 12 hours pass and upgrade is rescheduled for 5th app - assertEquals(0, tester.buildSystem().jobs().size()); - tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); - tester.upgrader().maintain(); - assertEquals(1, tester.buildSystem().jobs().size()); - assertEquals("Upgrade is rescheduled", DeploymentJobs.JobType.systemTest.id(), - tester.buildSystem().jobs().get(0).jobName()); - tester.deployCompletely(default4, applicationPackage); - assertEquals(version, tester.application(default4.id()).deployedVersion().get()); + assertEquals(version, tester.application(default0.id()).oldestDeployedVersion().get()); + assertEquals(version, tester.application(default1.id()).oldestDeployedVersion().get()); + assertEquals(version, tester.application(default2.id()).oldestDeployedVersion().get()); + assertEquals(version, tester.application(default3.id()).oldestDeployedVersion().get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json index 32d34edd576..425b9d4512d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json @@ -4,7 +4,7 @@ "validationOverrides": "<deployment version='1.0'/>", "deployments": [], "deploymentJobs": { - "projectId": 0, + "projectId": 42, "jobStatus": [ { "jobType": "system-test", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index b38a38c3120..2c1471b29b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -13,6 +13,8 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -69,10 +71,10 @@ public class ApplicationSerializerTest { List<JobStatus> statusList = new ArrayList<>(); statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest) - .withTriggering(37, Version.fromString("5.6.7"), Optional.empty(), true, "Test", Instant.ofEpochMilli(7)) + .withTriggering(Version.fromString("5.6.7"), Optional.empty(), true, "Test", Instant.ofEpochMilli(7)) .withCompletion(30, Optional.empty(), Instant.ofEpochMilli(8), tester.controller())); statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest) - .withTriggering(12, Version.fromString("5.6.6"), Optional.empty(), true, "Test 2", Instant.ofEpochMilli(5)) + .withTriggering(Version.fromString("5.6.6"), Optional.empty(), true, "Test 2", Instant.ofEpochMilli(5)) .withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller())); DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty()); @@ -82,7 +84,9 @@ public class ApplicationSerializerTest { validationOverrides, deployments, deploymentJobs, Optional.of(new Change.VersionChange(Version.fromString("6.7"))), - true); + true, + Optional.of(IssueId.from("1234")), + new MetricsService.ApplicationMetrics(0.5, 0.9)); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -105,10 +109,11 @@ public class ApplicationSerializerTest { serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest)); assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest)); - assertEquals(original.deploymentJobs().failingSince(), serialized.deploymentJobs().failingSince()); assertEquals(original.hasOutstandingChange(), serialized.hasOutstandingChange()); + assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId()); + assertEquals(original.deploying(), serialized.deploying()); // Test cluster utilization @@ -129,6 +134,9 @@ public class ApplicationSerializerTest { assertEquals(50, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE); // Test metrics + assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE); + assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE); + assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE); assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE); assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE); @@ -199,14 +207,11 @@ public class ApplicationSerializerTest { Application application = applicationSerializer.fromSlime(applicationSlime(false)); assertFalse(application.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).lastCompleted().get().upgrade()); } - - // TODO: Remove after October 2017 + @Test - public void testLegacySerializationWithZeroProjectId() { - Application original = applicationSerializer.fromSlime(applicationSlime(0, false)); - assertFalse(original.deploymentJobs().projectId().isPresent()); - Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); - assertFalse(serialized.deploymentJobs().projectId().isPresent()); + public void testCompleteApplicationDeserialization() { + Application application = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(longApplicationJson.getBytes(StandardCharsets.UTF_8))); + // ok if no error } private Slime applicationSlime(boolean error) { @@ -245,4 +250,6 @@ public class ApplicationSerializerTest { " }\n" + "}\n"; } + + private final String longApplicationJson = "{\"id\":\"tripod:service-aggregation-vespa:default\",\"deploymentSpecField\":\"<deployment version='1.0'>\\n <test />\\n <!--<staging />-->\\n <prod global-service-id=\\\"tripod\\\">\\n <region active=\\\"true\\\">us-east-3</region>\\n <region active=\\\"true\\\">us-west-1</region>\\n </prod>\\n</deployment>\\n\",\"validationOverrides\":\"<validation-overrides>\\n <allow until=\\\"2016-04-28\\\" comment=\\\"Renaming content cluster\\\">content-cluster-removal</allow>\\n <allow until=\\\"2016-08-22\\\" comment=\\\"Migrating us-east-3 to C-2E\\\">cluster-size-reduction</allow>\\n <allow until=\\\"2017-06-30\\\" comment=\\\"Test Vespa upgrade tests\\\">force-automatic-tenant-upgrade-test</allow>\\n</validation-overrides>\\n\",\"deployments\":[{\"zone\":{\"environment\":\"prod\",\"region\":\"us-west-1\"},\"version\":\"6.173.62\",\"deployTime\":1510837817704,\"applicationPackageRevision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"clusterInfo\":{\"tripod\":{\"flavor\":\"d-3-16-100\",\"cost\":9,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"container\",\"hostnames\":[\"oxy-oxygen-2001-4998-c-2942--10d1.gq1.yahoo.com\",\"oxy-oxygen-2001-4998-c-2942--10e2.gq1.yahoo.com\"]},\"tripodaggregation\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"oxy-oxygen-2001-4998-c-2941--106a.gq1.yahoo.com\",\"zt74700-v6-23.ostk.bm2.prod.gq1.yahoo.com\",\"zt74714-v6-28.ostk.bm2.prod.gq1.yahoo.com\",\"zt74730-v6-13.ostk.bm2.prod.gq1.yahoo.com\",\"zt74717-v6-7.ostk.bm2.prod.gq1.yahoo.com\",\"2080260-v6-12.ostk.bm2.prod.gq1.yahoo.com\",\"zt74719-v6-23.ostk.bm2.prod.gq1.yahoo.com\",\"zt74722-v6-26.ostk.bm2.prod.gq1.yahoo.com\",\"zt74704-v6-9.ostk.bm2.prod.gq1.yahoo.com\",\"oxy-oxygen-2001-4998-c-2942--107d.gq1.yahoo.com\"]},\"tripodaggregationstream\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt74727-v6-21.ostk.bm2.prod.gq1.yahoo.com\",\"zt74773-v6-8.ostk.bm2.prod.gq1.yahoo.com\",\"zt74699-v6-25.ostk.bm2.prod.gq1.yahoo.com\",\"zt74766-v6-27.ostk.bm2.prod.gq1.yahoo.com\"]}},\"clusterUtils\":{\"tripod\":{\"cpu\":0.1720353499228221,\"mem\":0.4986146831512451,\"disk\":0.0617671330041831,\"diskbusy\":0},\"tripodaggregation\":{\"cpu\":0.07505730001866318,\"mem\":0.7936344432830811,\"disk\":0.2260549694485994,\"diskbusy\":0},\"tripodaggregationstream\":{\"cpu\":0.01712671480989384,\"mem\":0.0225852754983035,\"disk\":0.006084436856721915,\"diskbusy\":0}},\"metrics\":{\"queriesPerSecond\":1.25,\"writesPerSecond\":43.83199977874756,\"documentCount\":525880277.9999999,\"queryLatencyMillis\":5.607503938674927,\"writeLatencyMillis\":20.57866265104621}},{\"zone\":{\"environment\":\"test\",\"region\":\"us-east-1\"},\"version\":\"6.173.62\",\"deployTime\":1511256872316,\"applicationPackageRevision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"clusterInfo\":{},\"clusterUtils\":{},\"metrics\":{\"queriesPerSecond\":0,\"writesPerSecond\":0,\"documentCount\":0,\"queryLatencyMillis\":0,\"writeLatencyMillis\":0}},{\"zone\":{\"environment\":\"dev\",\"region\":\"us-east-1\"},\"version\":\"6.173.62\",\"deployTime\":1510597489464,\"applicationPackageRevision\":{\"applicationPackageHash\":\"59b883f263c2a3c23dfab249730097d7e0e1ed32\"},\"clusterInfo\":{\"tripod\":{\"flavor\":\"d-2-8-50\",\"cost\":5,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"container\",\"hostnames\":[\"zt40807-v6-29.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregation\":{\"flavor\":\"d-2-8-50\",\"cost\":5,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40807-v6-24.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregationstream\":{\"flavor\":\"d-2-8-50\",\"cost\":5,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40694-v6-21.ostk.bm2.prod.bf1.yahoo.com\"]}},\"clusterUtils\":{\"tripod\":{\"cpu\":0.191833330678661,\"mem\":0.4625738318415235,\"disk\":0.05582004563850269,\"diskbusy\":0},\"tripodaggregation\":{\"cpu\":0.2227037978608054,\"mem\":0.2051752598416401,\"disk\":0.05471533698695047,\"diskbusy\":0},\"tripodaggregationstream\":{\"cpu\":0.1869410834020498,\"mem\":0.1691722576000564,\"disk\":0.04977374774258153,\"diskbusy\":0}},\"metrics\":{\"queriesPerSecond\":0,\"writesPerSecond\":0,\"documentCount\":30916,\"queryLatencyMillis\":0,\"writeLatencyMillis\":0}},{\"zone\":{\"environment\":\"prod\",\"region\":\"us-east-3\"},\"version\":\"6.173.62\",\"deployTime\":1510817190016,\"applicationPackageRevision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"clusterInfo\":{\"tripod\":{\"flavor\":\"d-3-16-100\",\"cost\":9,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"container\",\"hostnames\":[\"zt40738-v6-13.ostk.bm2.prod.bf1.yahoo.com\",\"zt40783-v6-31.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregation\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40819-v6-7.ostk.bm2.prod.bf1.yahoo.com\",\"zt40661-v6-3.ostk.bm2.prod.bf1.yahoo.com\",\"zt40805-v6-30.ostk.bm2.prod.bf1.yahoo.com\",\"zt40702-v6-32.ostk.bm2.prod.bf1.yahoo.com\",\"zt40706-v6-3.ostk.bm2.prod.bf1.yahoo.com\",\"zt40691-v6-27.ostk.bm2.prod.bf1.yahoo.com\",\"zt40676-v6-15.ostk.bm2.prod.bf1.yahoo.com\",\"zt40788-v6-23.ostk.bm2.prod.bf1.yahoo.com\",\"zt40782-v6-30.ostk.bm2.prod.bf1.yahoo.com\",\"zt40802-v6-32.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregationstream\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40779-v6-27.ostk.bm2.prod.bf1.yahoo.com\",\"zt40791-v6-15.ostk.bm2.prod.bf1.yahoo.com\",\"zt40733-v6-31.ostk.bm2.prod.bf1.yahoo.com\",\"zt40724-v6-30.ostk.bm2.prod.bf1.yahoo.com\"]}},\"clusterUtils\":{\"tripod\":{\"cpu\":0.2295038983007097,\"mem\":0.4627357390237263,\"disk\":0.05559941525894966,\"diskbusy\":0},\"tripodaggregation\":{\"cpu\":0.05340429087579549,\"mem\":0.8107630891552372,\"disk\":0.226444914138854,\"diskbusy\":0},\"tripodaggregationstream\":{\"cpu\":0.02148227413975218,\"mem\":0.02162174219104161,\"disk\":0.006057760545243265,\"diskbusy\":0}},\"metrics\":{\"queriesPerSecond\":1.734000012278557,\"writesPerSecond\":44.59999895095825,\"documentCount\":525868193.9999999,\"queryLatencyMillis\":5.65284947195106,\"writeLatencyMillis\":17.34593812832452}}],\"deploymentJobs\":{\"projectId\":102889,\"jobStatus\":[{\"jobType\":\"staging-test\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"system-test completed\",\"at\":1510830134259},\"lastCompleted\":{\"id\":1184,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"system-test completed\",\"at\":1510830684960},\"lastSuccess\":{\"id\":1184,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"system-test completed\",\"at\":1510830684960}},{\"jobType\":\"component\",\"lastCompleted\":{\"id\":849,\"version\":\"6.174.156\",\"upgrade\":false,\"reason\":\"Application commit\",\"at\":1511217733555},\"lastSuccess\":{\"id\":849,\"version\":\"6.174.156\",\"upgrade\":false,\"reason\":\"Application commit\",\"at\":1511217733555}},{\"jobType\":\"production-us-east-3\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"staging-test completed\",\"at\":1510830685127},\"lastCompleted\":{\"id\":923,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"staging-test completed\",\"at\":1510837650046},\"lastSuccess\":{\"id\":923,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"staging-test completed\",\"at\":1510837650046}},{\"jobType\":\"production-us-west-1\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"production-us-east-3 completed\",\"at\":1510837650139},\"lastCompleted\":{\"id\":646,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"production-us-east-3 completed\",\"at\":1510843559162},\"lastSuccess\":{\"id\":646,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"production-us-east-3 completed\",\"at\":1510843559162}},{\"jobType\":\"system-test\",\"jobError\":\"unknown\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"upgrade\":false,\"reason\":\"Available change in component\",\"at\":1511256608649},\"lastCompleted\":{\"id\":1686,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"upgrade\":false,\"reason\":\"Available change in component\",\"at\":1511256603353},\"firstFailing\":{\"id\":1659,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"upgrade\":false,\"reason\":\"component completed\",\"at\":1511219070725},\"lastSuccess\":{\"id\":1658,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"Upgrading to 6.173.62\",\"at\":1511175754163}}]},\"deployingField\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"outstandingChangeField\":false,\"queryQuality\":100,\"writeQuality\":99.99894341115082}"; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index f5f43265cb8..189b3a97a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -27,6 +27,8 @@ public class VersionStatusSerializerTest { Version.fromString("5.0"), Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")), + Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"), ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java new file mode 100644 index 00000000000..04a987d98d1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -0,0 +1,83 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Haakon Dybdahl + */ +public class ProxyRequestTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testEmpty() throws Exception { + exception.expectMessage("Request not set."); + testRequest(null, "/zone/v2/"); + } + + @Test + public void testBadUri() throws Exception { + exception.expectMessage("Request not starting with /zone/v2/"); + testRequest(URI.create("http://foo"), "/zone/v2/"); + } + + @Test + public void testConfigRequestEmpty() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar"), "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertEquals("bar", proxyRequest.getRegion()); + assertFalse(proxyRequest.isDiscoveryRequest()); + assertTrue(proxyRequest.getConfigServerRequest().isEmpty()); + + } + + @Test + public void testDiscoveryRequest() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo"), "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertTrue(proxyRequest.isDiscoveryRequest()); + + } + + @Test + public void testProxyRequest() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar/bla/bla/v1/something"), + "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertEquals("/bla/bla/v1/something", proxyRequest.getConfigServerRequest()); + } + + @Test + public void testProxyRequestWithParameters() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar/something?p=v&q=y"), + "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertEquals("/something?p=v&q=y", proxyRequest.getConfigServerRequest()); + } + + private static ProxyRequest testRequest(URI url, String pathPrefix) throws IOException, ProxyException { + return new ProxyRequest(url, headers("controller:49152"), null, "GET", pathPrefix); + } + + private static Map<String, List<String>> headers(String hostPort) { + Map<String, List<String>> headers = new HashMap<>(); + headers.put("host", Collections.singletonList(hostPort)); + return Collections.unmodifiableMap(headers); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java new file mode 100644 index 00000000000..8dbd1c4ef61 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author Haakon Dybdahl + */ +public class ProxyResponseTest { + + @Test + public void testRewriteUrl() throws Exception { + String controllerPrefix = "/zone/v2/"; + URI configServer = URI.create("http://configserver:1234"); + ProxyRequest request = new ProxyRequest(URI.create("http://foo/zone/v2/env/region/configserver"), + headers("controller:49152"), null, "GET", + controllerPrefix); + ProxyResponse proxyResponse = new ProxyResponse( + request, + "response link is http://configserver:1234/bla/bla/", + 200, + Optional.of(configServer), + "application/json"); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + proxyResponse.render(outputStream); + String document = new String(outputStream.toByteArray(),"UTF-8"); + assertEquals("response link is http://controller:49152/zone/v2/env/region/bla/bla/", document); + } + + @Test + public void testRewriteSecureUrl() throws Exception { + String controllerPrefix = "/zone/v2/"; + URI configServer = URI.create("http://configserver:1234"); + ProxyRequest request = new ProxyRequest(URI.create("https://foo/zone/v2/env/region/configserver"), + headers("controller:49152"), null, "GET", + controllerPrefix); + ProxyResponse proxyResponse = new ProxyResponse( + request, + "response link is http://configserver:1234/bla/bla/", + 200, + Optional.of(configServer), + "application/json"); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + proxyResponse.render(outputStream); + String document = new String(outputStream.toByteArray(),"UTF-8"); + assertEquals("response link is https://controller:49152/zone/v2/env/region/bla/bla/", document); + } + + private static Map<String, List<String>> headers(String hostPort) { + Map<String, List<String>> headers = new HashMap<>(); + headers.put("host", Collections.singletonList(hostPort)); + return Collections.unmodifiableMap(headers); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 45a8972eafe..6c5120df515 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -88,6 +88,12 @@ public class ContainerControllerTester { } public void notifyJobCompletion(ApplicationId applicationId, long projectId, boolean success, DeploymentJobs.JobType job) { + try { + Thread.sleep(1); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } controller().applications().notifyJobCompletion(new DeploymentJobs.JobReport(applicationId, job, projectId, 42, success ? Optional.empty() : Optional.of(DeploymentJobs.JobError.unknown) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index b55ee9a195f..c0e8b48f821 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Supplier; import static org.junit.Assert.assertEquals; @@ -52,10 +53,18 @@ public class ContainerTester { controller.updateVersionStatus(VersionStatus.compute(controller, version)); } + public void assertResponse(Supplier<Request> request, File responseFile) throws IOException { + assertResponse(request.get(), responseFile); + } + public void assertResponse(Request request, File responseFile) throws IOException { assertResponse(request, responseFile, 200); } + public void assertResponse(Supplier<Request> request, File responseFile, int expectedStatusCode) throws IOException { + assertResponse(request.get(), responseFile, expectedStatusCode); + } + public void assertResponse(Request request, File responseFile, int expectedStatusCode) throws IOException { String expectedResponse = IOUtils.readFile(new File(responseFilePath + responseFile.toString())); expectedResponse = include(expectedResponse); @@ -72,10 +81,18 @@ public class ContainerTester { replace(new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8), replaceStrings)); } + public void assertResponse(Supplier<Request> request, String expectedResponse) throws IOException { + assertResponse(request.get(), expectedResponse, 200); + } + public void assertResponse(Request request, String expectedResponse) throws IOException { assertResponse(request, expectedResponse, 200); } + public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) throws IOException { + assertResponse(request.get(), expectedResponse, expectedStatusCode); + } + public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException { Response response = container.handleRequest(request); assertEquals(expectedResponse, response.getBodyAsString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index e6c0ce9027d..044c5d75d12 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -43,10 +43,12 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization'/>" + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerClientMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.ZoneRegistryMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.Controller'/>" + + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerProxyMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.integration.MockMetricsService'/>" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>" + @@ -69,6 +71,14 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.screwdriver.ScrewdriverApiHandler'>" + " <binding>http://*/screwdriver/v1/*</binding>" + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>" + + " <binding>http://*/zone/v1</binding>" + + " <binding>http://*/zone/v1/*</binding>" + + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>" + + " <binding>http://*/zone/v2</binding>" + + " <binding>http://*/zone/v2/*</binding>" + + " </handler>" + "</jdisc>"; protected void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e3443d6c014..bf4586f9fd0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -5,14 +5,14 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; -import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -47,6 +47,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; + +import static com.yahoo.application.container.handler.Request.Method.DELETE; +import static com.yahoo.application.container.handler.Request.Method.GET; +import static com.yahoo.application.container.handler.Request.Method.POST; +import static com.yahoo.application.container.handler.Request.Method.PUT; /** * @author bratseth @@ -72,63 +78,87 @@ public class ApplicationApiTest extends ControllerContainerTest { addTenantAthenzDomain(athenzUserDomain, "mytenant"); // (Necessary but not provided in this API) // GET API root - tester.assertResponse(request("/application/v4/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/", GET), new File("root.json")); // GET athens domains - tester.assertResponse(request("/application/v4/athensDomain/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/athensDomain/", GET), new File("athensDomain-list.json")); // GET OpsDB properties - tester.assertResponse(request("/application/v4/property/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/property/", GET), new File("property-list.json")); // GET cookie freshness - tester.assertResponse(request("/application/v4/cookiefreshness/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/cookiefreshness/", GET), new File("cookiefreshness.json")); // POST (add) a tenant without property ID - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // PUT (modify) a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.PUT), + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // GET the authenticated user (with associated tenants) - tester.assertResponse(request("/application/v4/user", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/user", GET), new File("user.json")); // GET all tenants - tester.assertResponse(request("/application/v4/tenant/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/", GET), new File("tenant-list.json")); + + + // Add another Athens domain, so we can try to create more tenants + addTenantAthenzDomain("domain2", "mytenant"); // New domain to test tenant w/property ID + // Add property info for that property id, as well, in the mock organization. + addPropertyData((MockOrganization) controllerTester.controller().organization(), "1234"); + // POST (add) a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", POST) + .data("{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}"), + new File("tenant-without-applications-with-id.json")); + // PUT (modify) a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", PUT) + .data("{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}"), + new File("tenant-without-applications-with-id.json")); + // GET a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", GET), + new File("tenant-without-applications-with-id.json")); + + // Test legacy OpsDB tenants + // POST (add) an OpsDB tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant3", POST) + .data("{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}"), + new File("opsdb-tenant-with-id-without-applications.json")); + // PUT (modify) the OpsDB tenant to set another property + tester.assertResponse(request("/application/v4/tenant/tenant3", PUT) + .data("{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}"), + new File("opsdb-tenant-with-new-id-without-applications.json")); + // POST (create) an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // GET a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1", GET), new File("tenant-with-application.json")); // GET tenant applications - tester.assertResponse(request("/application/v4/tenant/tenant1/application/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET), new File("application-list.json")); // POST triggering of a full deployment to an application (if version is omitted, current system version is used) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", POST) + .data("6.1.0"), new File("application-deployment.json")); // DELETE (cancel) ongoing change - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE), new File("application-deployment-cancelled.json")); // DELETE (cancel) again is a no-op - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE), new File("application-deployment-cancelled-no-op.json")); // POST (deploy) an application to a zone - manual user deployment HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), new File("deploy-result.json")); // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. @@ -138,168 +168,146 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // (Necessary but not provided in this API) // Trigger deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", POST) + .data("6.1.0"), new File("application-deployment.json")); // ... systemtest - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", - createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default"); controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.systemTest); // Called through the separate screwdriver/v1 API // ... staging - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", - createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default"); controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.stagingTest); // ... prod zone - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", - createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, screwdriverProjectId, false, DeploymentJobs.JobType.productionCorpUsEast1); // GET tenant screwdriver projects - tester.assertResponse(request("/application/v4/tenant-pipeline/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant-pipeline/", GET), new File("tenant-pipelines.json")); + setDeploymentMaintainedInfo(controllerTester); // GET tenant application deployments - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET), new File("application.json")); // GET an application deployment - setDeploymentMaintainedInfo(controllerTester); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", GET), new File("deployment.json")); + + addIssues(controllerTester, ApplicationId.from("tenant1", "application1", "default")); + // GET at root, with "&recursive=deployment", returns info about all tenants, their applications and their deployments + tester.assertResponse(request("/application/v4/", GET) + .domain("domain1").user("mytenant") + .recursive("deployment"), + new File("recursive-root.json")); + // GET at root, with "&recursive=tenant", returns info about all tenants, with limmited info about their applications. + tester.assertResponse(request("/application/v4/", GET) + .domain("domain1").user("mytenant") + .recursive("tenant"), + new File("recursive-until-tenant-root.json")); + // GET at a tenant, with "&recursive=true", returns full info about their applications and their deployments + tester.assertResponse(request("/application/v4/tenant/tenant1/", GET) + .domain("domain1").user("mytenant") + .recursive("true"), + new File("tenant1-recursive.json")); + // GET at an application, with "&recursive=true", returns full info about its deployments + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/", GET) + .domain("domain1").user("mytenant") + .recursive("true"), + new File("application1-recursive.json")); + + // POST a 'restart application' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", POST), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // POST a 'restart application' command with a host filter (other filters not supported yet) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", POST), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // POST a 'log' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/log", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/log", POST), new File("log-response.json")); // Proxied to config server, not sure about the expected return format // GET (wait for) convergence - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", GET), new File("convergence.json")); // GET services - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", GET), new File("services.json")); // GET service - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET), new File("service.json")); // DELETE (deactivate) a deployment - dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); // DELETE (deactivate) a deployment - prod - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // DELETE (deactivate) a deployment is idempotent - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // DELETE an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE), ""); // DELETE a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), new File("tenant-without-applications.json")); // PUT (create) the authenticated user - tester.assertResponse(request("/application/v4/user?user=newuser&domain=by", - new byte[0], - Request.Method.PUT, - athenzUserDomain, "newuser", "application/json"), + byte[] data = new byte[0]; + tester.assertResponse(request("/application/v4/user?user=newuser&domain=by", PUT) + .data(data) + .domain(athenzUserDomain).user("newuser"), new File("create-user-response.json")); // OPTIONS return 200 OK - tester.assertResponse(request("/application/v4/", "", Request.Method.OPTIONS), + tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS), ""); - // Add another Athens domain, so we can try to create more tenants - addTenantAthenzDomain("domain2", "mytenant"); // New domain to test tenant w/property ID - // Add property info for that property id, as well, in the mock organization. - addPropertyData((MockOrganization) controllerTester.controller().organization(), "1234"); - // POST (add) a tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant2", - "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}", - Request.Method.POST), - new File("tenant-without-applications-with-id.json")); - // PUT (modify) a tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant2", - "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}", - Request.Method.PUT), - new File("tenant-without-applications-with-id.json")); - // GET a tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant2", "", Request.Method.GET), - new File("tenant-without-applications-with-id.json")); - - // Test legacy OpsDB tenants - // POST (add) an OpsDB tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant3", - "{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}", - Request.Method.POST), - new File("opsdb-tenant-with-id-without-applications.json")); - // PUT (modify) the OpsDB tenant to set another property - tester.assertResponse(request("/application/v4/tenant/tenant3", - "{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}", - Request.Method.PUT), - new File("opsdb-tenant-with-new-id-without-applications.json")); - // GET global rotation status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", GET), new File("global-rotation.json")); // GET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", "", Request.Method.GET), - new File("global-rotation-get.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", GET), + new File("global-rotation-get.json")); // SET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.PUT), - new File("global-rotation-put.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", PUT) + .data("{\"reason\":\"because i can\"}"), + new File("global-rotation-put.json")); // DELETE global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.DELETE), - new File("global-rotation-delete.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", DELETE) + .data("{\"reason\":\"because i can\"}"), + new File("global-rotation-delete.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", "", Request.Method.POST), - "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), - "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", POST), + "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", POST), + "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); controllerTester.controller().deconstruct(); } - private void addPropertyData(MockOrganization organization, String propertyIdValue) { - PropertyId propertyId = new PropertyId(propertyIdValue); - organization.addProperty(propertyId); - organization.setContactsFor(propertyId, Arrays.asList(Collections.singletonList(User.from("alice")), - Collections.singletonList(User.from("bob")))); + private void addIssues(ContainerControllerTester tester, ApplicationId id) { + tester.controller().applications().lockedOrThrow(id, application -> + tester.controller().applications().store(application + .withDeploymentIssueId(IssueId.from("123")) + .withOwnershipIssueId(IssueId.from("321")))); } @Test @@ -312,23 +320,19 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // Create tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // POST (deploy) an application to a prod zone - allowed when project ID is not specified HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); } @@ -342,15 +346,12 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // Create tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // Deploy @@ -364,10 +365,9 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, deployData); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsEast3); @@ -381,22 +381,20 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, deployData); // us-west-1 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsWest1); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + .data(deployData).domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsEast3); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), + setDeploymentMaintainedInfo(controllerTester); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET), new File("application-without-change-multiple-deployments.json")); } @@ -407,63 +405,49 @@ public class ApplicationApiTest extends ControllerContainerTest { addTenantAthenzDomain("domain1", "mytenant"); // PUT (update) non-existing tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.PUT), + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); // GET non-existing tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "", - Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1", GET), "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); // GET non-existing application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET), "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}", 404); // GET non-existing deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east/instance/default", - "", - Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east/instance/default", GET), "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}", 404); // POST (add) a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // POST (add) another tenant under the same domain - tester.assertResponse(request("/application/v4/tenant/tenant2", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant2", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create tenant 'tenant2': The Athens domain 'domain1' is already connected to tenant 'tenant1'\"}", 400); // Add the same tenant again - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", 400); // POST (create) an (empty) application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // Create the same application again - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"An application with id 'tenant1.application1' already exists\"}", 400); @@ -472,64 +456,56 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application with an invalid application package HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), new File("deploy-failure.json"), 400); // POST (deploy) an application without available capacity configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), new File("deploy-out-of-capacity.json"), 400); // POST (deploy) an application where activation fails configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to activate application", ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), - new File("deploy-activation-conflict.json"), 409); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), + new File("deploy-activation-conflict.json"), 409); // POST (deploy) an application where we get an internal server error configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Internal server error", ConfigServerException.ErrorCode.INTERNAL_SERVER_ERROR, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), - new File("deploy-internal-server-error.json"), 500); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), + new File("deploy-internal-server-error.json"), 500); // DELETE tenant which has an application - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not delete tenant 'tenant1': This tenant has active applications\"}", 400); // DELETE application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE), ""); // DELETE application again - should produce 404 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", 404); // DELETE tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), new File("tenant-without-applications.json")); // DELETE tenant again - should produce 404 - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete tenant 'tenant1': Tenant not found\"}", + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete tenant 'tenant1': Tenant not found\"}", 404); // Promote application chef env for nonexistent tenant/application - tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), - "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Unable to promote Chef environments for application\"}", - 500); + tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", POST), + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Unable to promote Chef environments for application\"}", + 500); } @Test @@ -539,102 +515,85 @@ public class ApplicationApiTest extends ControllerContainerTest { String unauthorizedUser = "othertenant"; // Mutation without an authorized user is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST, - "domain1", null), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(null), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User is not authenticated\"}", 403); // ... but read methods are allowed - tester.assertResponse(request("/application/v4/tenant/", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.GET, - "domain1", null), + tester.assertResponse(request("/application/v4/tenant/", GET) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(null), "[]", 200); addTenantAthenzDomain("domain1", "mytenant"); // Creating a tenant for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"The user 'othertenant' is not admin in Athenz domain 'domain1'\"}", 403); // (Create it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST, - "domain1", authorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(authorizedUser), new File("tenant-without-applications.json"), 200); // Creating an application for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); // (Create it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST, - "domain1", authorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + .domain("domain1").user(authorizedUser), new File("application-reference.json"), 200); // Deploy to an authorized zone by a user tenant is disallowed HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), "{\"error-code\":\"FORBIDDEN\",\"message\":\"Principal 'mytenant' is not a Screwdriver principal. Excepted principal with Athenz domain 'cd.screwdriver.project', got 'domain1'.\"}", 403); // Deleting an application for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); // (Deleting it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE, - "domain1", authorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + .domain("domain1").user(authorizedUser), "", 200); // Updating a tenant for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.PUT, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); // Change Athens domain addTenantAthenzDomain("domain2", "mytenant"); - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain2\", \"property\":\"property1\"}", - Request.Method.PUT, - "domain1", authorizedUser), - "{\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}", + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain2\", \"property\":\"property1\"}") + .domain("domain1").user(authorizedUser), + "{\"tenant\":\"tenant1\",\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}", 200); // Deleting a tenant for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "", - Request.Method.DELETE, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); } @@ -669,33 +628,53 @@ public class ApplicationApiTest extends ControllerContainerTest { "}"; } - - /** Make a request with (athens) user domain1.mytenant1 */ - private Request request(String path, String data, Request.Method method) { - return request(path, data.getBytes(StandardCharsets.UTF_8), method, "domain1", "mytenant", "application/json"); - } - private Request request(String path, String data, Request.Method method, String domain, String user) { - return request(path, data.getBytes(StandardCharsets.UTF_8), method, domain, user, "application/json"); - } + private static class RequestBuilder implements Supplier<Request> { - private Request request(String path, byte[] data, Request.Method method, String domain, String user, String contentType) { - // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters - Request request = new Request("http://localhost:8080" + path + "?domain=" + domain + - (user != null ? "&user=" + user : ""), - data, method); - request.getHeaders().put("Content-Type", contentType); - return request; - } + private final String path; + private final Request.Method method; + private byte[] data = new byte[0]; + private String domain = "domain1"; + private String user = "mytenant"; + private String contentType = "application/json"; + private String recursive; - private Request request(String path, HttpEntity data, Request.Method method, String domain, String user) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - data.writeTo(out); - } catch (IOException e) { - throw new UncheckedIOException(e); + private RequestBuilder(String path, Request.Method method) { + this.path = path; + this.method = method; } - return request(path, out.toByteArray(), method, domain, user, data.getContentType().getValue()); + + private RequestBuilder data(byte[] data) { this.data = data; return this; } + private RequestBuilder data(String data) { return data(data.getBytes(StandardCharsets.UTF_8)); } + private RequestBuilder data(HttpEntity data) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + data.writeTo(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return data(out.toByteArray()).contentType(data.getContentType().getValue()); + } + private RequestBuilder domain(String domain) { this.domain = domain; return this; } + private RequestBuilder user(String user) { this.user = user; return this; } + private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } + private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } + + @Override + public Request get() { + Request request = new Request("http://localhost:8080" + path + + // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters + "?domain=" + domain + (user == null ? "" : "&user=" + user) + + (recursive == null ? "" : "&recursive=" + recursive), + data, method); + request.getHeaders().put("Content-Type", contentType); + return request; + } + } + + /** Make a request with (athens) user domain1.mytenant */ + private RequestBuilder request(String path, Request.Method method) { + return new RequestBuilder(path, method); } /** @@ -734,34 +713,28 @@ public class ApplicationApiTest extends ControllerContainerTest { // system-test String testPath = String.format("/application/v4/tenant/%s/application/%s/environment/test/region/us-east-1/instance/default", application.tenant().value(), application.application().value()); - tester.assertResponse(request(testPath, - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), - new File("deploy-result.json")); - tester.assertResponse(request(testPath, - "", - Request.Method.DELETE), + tester.assertResponse(request(testPath, POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), + new File("deploy-result.json")); + tester.assertResponse(request(testPath, DELETE), "Deactivated " + testPath.replaceFirst("/application/v4/", "")); controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.systemTest); // staging String stagingPath = String.format("/application/v4/tenant/%s/application/%s/environment/staging/region/us-east-3/instance/default", application.tenant().value(), application.application().value()); - tester.assertResponse(request(stagingPath, - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), - new File("deploy-result.json")); - tester.assertResponse(request(stagingPath, - "", - Request.Method.DELETE), + tester.assertResponse(request(stagingPath, POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), + new File("deploy-result.json")); + tester.assertResponse(request(stagingPath, DELETE), "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.stagingTest); } /** - * Cluster info, utilization and deployment metrics are maintained async by maintainers. + * Cluster info, utilization and application and deployment metrics are maintained async by maintainers. * * This sets these values as if the maintainers has been ran. * @@ -769,9 +742,9 @@ public class ApplicationApiTest extends ControllerContainerTest { */ private void setDeploymentMaintainedInfo(ContainerControllerTester controllerTester) { for (Application application : controllerTester.controller().applications().asList()) { - try (Lock lock = controllerTester.controller().applications().lock(application.id())) { - LockedApplication lockedApplication = controllerTester.controller().applications() - .require(application.id(), lock); + controllerTester.controller().applications().lockedOrThrow(application.id(), lockedApplication -> { + lockedApplication = lockedApplication.with(new ApplicationMetrics(0.5, 0.7)); + for (Deployment deployment : application.deployments().values()) { Map<ClusterSpec.Id, ClusterInfo> clusterInfo = new HashMap<>(); List<String> hostnames = new ArrayList<>(); @@ -780,13 +753,23 @@ public class ApplicationApiTest extends ControllerContainerTest { clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); Map<ClusterSpec.Id, ClusterUtilization> clusterUtils = new HashMap<>(); clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); - deployment = deployment.withClusterInfo(clusterInfo); - deployment = deployment.withClusterUtils(clusterUtils); - deployment = deployment.withMetrics(new DeploymentMetrics(1,2,3,4,5)); - controllerTester.controller().applications().store(lockedApplication.with(deployment)); + DeploymentMetrics metrics = new DeploymentMetrics(1,2,3,4,5); + + lockedApplication = lockedApplication + .withClusterInfo(deployment.zone(), clusterInfo) + .withClusterUtilization(deployment.zone(), clusterUtils) + .with(deployment.zone(), metrics); } - } + controllerTester.controller().applications().store(lockedApplication); + }); } } + private void addPropertyData(MockOrganization organization, String propertyIdValue) { + PropertyId propertyId = new PropertyId(propertyIdValue); + organization.addProperty(propertyId); + organization.setContactsFor(propertyId, Arrays.asList(Collections.singletonList(User.from("alice")), + Collections.singletonList(User.from("bob")))); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index fe9c373b7d5..6442ddf5c02 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -1,4 +1,6 @@ { + "application": "application1", + "instance": "default", "deploymentJobs": [ { "type": "component", @@ -30,7 +32,7 @@ "gitCommit": "commit1" } }, - "reason": "component completed successfully in build 42", + "reason": "component completed", "at": "(ignore)" }, "lastCompleted": { @@ -44,7 +46,7 @@ "gitCommit": "commit1" } }, - "reason": "component completed successfully in build 42", + "reason": "component completed", "at": "(ignore)" }, "lastSuccess": { @@ -58,7 +60,7 @@ "gitCommit": "commit1" } }, - "reason": "component completed successfully in build 42", + "reason": "component completed", "at": "(ignore)" } }, @@ -76,7 +78,7 @@ "gitCommit": "commit1" } }, - "reason":"systemTest completed successfully in build 42", + "reason":"system-test completed", "at": "(ignore)" }, "lastCompleted": { @@ -90,7 +92,7 @@ "gitCommit": "commit1" } }, - "reason":"systemTest completed successfully in build 42", + "reason":"system-test completed", "at": "(ignore)" }, "lastSuccess": { @@ -104,7 +106,7 @@ "gitCommit": "commit1" } }, - "reason":"systemTest completed successfully in build 42", + "reason":"system-test completed", "at": "(ignore)" } }, @@ -122,7 +124,7 @@ "gitCommit": "commit1" } }, - "reason":"stagingTest completed successfully in build 42", + "reason":"staging-test completed", "at": "(ignore)" }, "lastCompleted": { @@ -136,7 +138,7 @@ "gitCommit": "commit1" } }, - "reason":"stagingTest completed successfully in build 42", + "reason":"staging-test completed", "at": "(ignore)" }, "lastSuccess": { @@ -150,7 +152,7 @@ "gitCommit": "commit1" } }, - "reason":"stagingTest completed successfully in build 42", + "reason":"staging-test completed", "at": "(ignore)" } }, @@ -168,7 +170,7 @@ "gitCommit": "commit1" } }, - "reason":"productionUsWest1 completed successfully in build 42", + "reason":"production-us-west-1 completed", "at": "(ignore)" }, "lastCompleted": { @@ -182,7 +184,7 @@ "gitCommit": "commit1" } }, - "reason":"productionUsWest1 completed successfully in build 42", + "reason":"production-us-west-1 completed", "at": "(ignore)" }, "lastSuccess": { @@ -196,7 +198,7 @@ "gitCommit": "commit1" } }, - "reason":"productionUsWest1 completed successfully in build 42", + "reason":"production-us-west-1 completed", "at": "(ignore)" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 3dca8103ed7..fdd3dcc4d5c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -1,4 +1,6 @@ { + "application": "application1", + "instance": "default", "deploying": { "version": "(ignore)" }, @@ -63,7 +65,7 @@ "gitCommit": "commit1" } }, - "reason": "systemTest completed successfully in build 42", + "reason": "system-test completed", "at": "(ignore)" }, "lastCompleted": { @@ -77,7 +79,7 @@ "gitCommit": "commit1" } }, - "reason": "systemTest completed successfully in build 42", + "reason": "system-test completed", "at": "(ignore)" }, "lastSuccess": { @@ -91,7 +93,7 @@ "gitCommit": "commit1" } }, - "reason": "systemTest completed successfully in build 42", + "reason": "system-test completed", "at": "(ignore)" } }, @@ -109,7 +111,7 @@ "gitCommit": "commit1" } }, - "reason": "Retrying as build 42 just started failing", + "reason": "Immediate retry on failure", "at": "(ignore)" }, "lastCompleted": { @@ -123,7 +125,7 @@ "gitCommit": "commit1" } }, - "reason": "stagingTest completed successfully in build 42", + "reason": "staging-test completed", "at": "(ignore)" }, "firstFailing": { @@ -137,7 +139,7 @@ "gitCommit": "commit1" } }, - "reason": "stagingTest completed successfully in build 42", + "reason": "staging-test completed", "at": "(ignore)" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json new file mode 100644 index 00000000000..41556c04209 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -0,0 +1,161 @@ +{ + "application": "application1", + "instance": "default", + "deploying": { + "version": "6.1" + }, + "deploymentJobs": [ + { + "type": "system-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "", + "at": "(ignore)" + } + }, + { + "type": "staging-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "system-test completed", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "system-test completed", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "system-test completed", + "at": "(ignore)" + } + }, + { + "type": "production-corp-us-east-1", + "success": false, + "lastTriggered": { + "id": -1, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Immediate retry on failure", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "staging-test completed", + "at": "(ignore)" + }, + "firstFailing": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "staging-test completed", + "at": "(ignore)" + } + } + ], + "compileVersion": "6.1.0", + "globalRotations": [ + "http://fake-global-rotation-tenant1.application1" + ], + "instances": [ + @include(dev-us-west-1.json), + @include(prod-corp-us-east-1.json) + ], + "metrics": { + "queryServiceQuality": 0.5, + "writeServiceQuality": 0.7 + }, + "ownershipIssueId": "321", + "deploymentIssueId": "123" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json new file mode 100644 index 00000000000..062f4408518 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json @@ -0,0 +1,63 @@ +{ + "environment": "dev", + "region": "us-west-1", + "instance": "default", + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080" + ], + "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-west-1&application=tenant1.application1", + "version": "6.1.0", + "revision": "(ignore)", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "123", + + + "cost": { + "tco": 74, + "waste": 0, + "utilization": 2.999999999999999, + "cluster": { + "cluster1": { + "count": 2, + "resource": "cpu", + "utilization": 2.999999999999999, + "tco": 74, + "waste": 0, + "flavor": "flavor1", + "flavorCost": 37.0, + "flavorCpu": 2.0, + "flavorMem": 4.0, + "flavorDisk": 50.0, + "type": "content", + "util": { + "cpu": 2.999999999999999, + "mem": 0.4285714285714286, + "disk": 0.5714285714285715, + "diskBusy": 1.0 + }, + "usage": { + "cpu": 0.6, + "mem": 0.3, + "disk": 0.4, + "diskBusy": 0.3 + }, + "hostnames": [ + "host1", + "host2" + ] + } + } + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0 + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json index 8acb4a045f3..a2e70d9c1eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json @@ -1,4 +1,5 @@ { + "tenant": "tenant3", "type": "OPSDB", "property": "property1", "propertyId": "1234", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json index 3f4b6017971..f9161ea49b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json @@ -1,4 +1,5 @@ { + "tenant": "tenant3", "type": "OPSDB", "property": "property2", "propertyId": "4321", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json new file mode 100644 index 00000000000..75b257da0ed --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json @@ -0,0 +1,68 @@ +{ + "environment": "prod", + "region": "corp-us-east-1", + "instance": "default", + "bcpStatus": { + "rotationStatus": "UNKNOWN" + }, + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080" + ], + "nodes": "http://localhost:8080/zone/v2/prod/corp-us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "elkUrl": "http://log.prod.corp-us-east-1.test/#/discover?_g=()&_a=(columns:!(_source),index:'logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'HV-tenant:%22tenant1%22%20AND%20HV-application:%22application1%22%20AND%20HV-region:%22corp-us-east-1%22%20AND%20HV-instance:%22default%22%20AND%20HV-environment:%22prod%22')),sort:!('@timestamp',desc))", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=corp-us-east-1&application=tenant1.application1", + "version": "6.1.0", + "revision": "(ignore)", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "123", + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1", + "cost": { + "tco": 74, + "waste": 0, + "utilization": 2.999999999999999, + "cluster": { + "cluster1": { + "count": 2, + "resource": "cpu", + "utilization": 2.999999999999999, + "tco": 74, + "waste": 0, + "flavor": "flavor1", + "flavorCost": 37.0, + "flavorCpu": 2.0, + "flavorMem": 4.0, + "flavorDisk": 50.0, + "type": "content", + "util": { + "cpu": 2.999999999999999, + "mem": 0.4285714285714286, + "disk": 0.5714285714285715, + "diskBusy": 1.0 + }, + "usage": { + "cpu": 0.6, + "mem": 0.3, + "disk": 0.4, + "diskBusy": 0.3 + }, + "hostnames": [ + "host1", + "host2" + ] + } + } + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json new file mode 100644 index 00000000000..a4395faede4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json @@ -0,0 +1,5 @@ +[ + @include(tenant2.json), + @include(tenant3.json), + @include(tenant1-recursive.json) +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json new file mode 100644 index 00000000000..35ed8181fac --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json @@ -0,0 +1,6 @@ +[ + @include(tenant2.json), + @include(tenant3.json), + @include(tenant-with-application.json) +] + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index 87901218c2e..ad8e65692b4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -1,4 +1,5 @@ { + "tenant": "tenant1", "type": "ATHENS", "athensDomain": "domain1", "property": "property1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json index ede2413218d..69949c47d8c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json @@ -1,4 +1,5 @@ { + "tenant": "tenant2", "type": "ATHENS", "athensDomain": "domain2", "property": "property2", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json index 69669b5dfb8..3ad5a307348 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json @@ -1,4 +1,5 @@ { + "tenant": "tenant1", "type": "ATHENS", "athensDomain": "domain1", "property": "property1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json new file mode 100644 index 00000000000..309177e6285 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json @@ -0,0 +1,9 @@ +{ + "tenant": "tenant1", + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1", + "applications": [ + @include(application1-recursive.json) + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json new file mode 100644 index 00000000000..6e66202b70d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json @@ -0,0 +1,19 @@ +{ + "tenant": "tenant2", + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property2", + "propertyId": "1234", + "applications": [], + "propertyUrl": "www.properties.tld/1234", + "contactsUrl": "www.contacts.tld/1234", + "issueCreationUrl": "www.issues.tld/1234", + "contacts": [ + [ + "alice" + ], + [ + "bob" + ] + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json new file mode 100644 index 00000000000..fdf3ca490f4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json @@ -0,0 +1,12 @@ +{ + "tenant": "tenant3", + "type": "OPSDB", + "property": "property2", + "propertyId": "4321", + "userGroup": "group1", + "applications": [], + "propertyUrl": "www.properties.tld/4321", + "contactsUrl": "www.contacts.tld/4321", + "issueCreationUrl": "www.issues.tld/4321", + "contacts": [] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 3633860772b..354bab4379c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -1,19 +1,16 @@ { "jobs": [ { - "name": "DelayedDeployer" + "name": "ApplicationOwnershipConfirmer" }, { - "name": "BlockedChangeDeployer" - }, - { - "name": "Upgrader" + "name": "ClusterInfoMaintainer" }, { - "name": "FailureRedeployer" + "name": "ClusterUtilizationMaintainer" }, { - "name": "VersionStatusUpdater" + "name": "DeploymentExpirer" }, { "name": "DeploymentIssueReporter" @@ -22,22 +19,22 @@ "name": "DeploymentMetricsMaintainer" }, { - "name": "OutstandingChangeDeployer" + "name": "MetricsReporter" }, { - "name": "ClusterUtilizationMaintainer" + "name": "OutstandingChangeDeployer" }, { - "name": "ClusterInfoMaintainer" + "name": "ReadyJobsTrigger" }, { - "name": "DeploymentExpirer" + "name": "Upgrader" }, { - "name": "MetricsReporter" + "name": "VersionStatusUpdater" } ], "inactive": [ "DeploymentExpirer" ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 7fd000b82c5..5f7fedfd75f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -1,27 +1,26 @@ { "versions":[ { - "version":"(ignore)", - "confidence":"high", - "commit":"(ignore)", - "date":0, - "controllerVersion":false, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - - ], - "productionApplications":[ + "version": "(ignore)", + "confidence": "high", + "commit": "(ignore)", + "date": 0, + "controllerVersion": false, + "systemVersion": false, + "configServers": [ ], + "failingApplications": [ ], + "productionApplications": [ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy":"default" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "productionJobs": 1, + "productionSuccesses": 1 } - ] + ], + "deployingApplications": [ ] }, { "version":"(ignore)", @@ -40,40 +39,47 @@ ], "failingApplications":[ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy":"default", - "failingSince":"(ignore)" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "failing": "staging-test" } ], "productionApplications":[ { - "tenant":"tenant2", - "application":"application2", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant2/application/application2", - "upgradePolicy":"default" + "tenant": "tenant2", + "application": "application2", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", + "upgradePolicy": "default", + "productionJobs": 1, + "productionSuccesses": 1 + } + ], + "deployingApplications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "running": "staging-test" } ] }, { - "version":"(ignore)", - "confidence":"normal", - "commit":"(ignore)", - "date":0, - "controllerVersion":true, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - - ], - "productionApplications":[ - - ] + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": true, + "systemVersion": false, + "configServers": [ ], + "failingApplications": [ ], + "productionApplications": [ ], + "deployingApplications": [ ] } ] }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 1638a2845ed..e6b3eacd44e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -96,14 +96,14 @@ public class ScrewdriverApiTest extends ControllerContainerTest { Response response; response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.GET)); - assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); - assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.jobName())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.jobName())); assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); // Check that GET didn't affect the enqueued jobs. response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE)); - assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); - assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.jobName())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.jobName())); assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); Thread.sleep(50); @@ -148,11 +148,8 @@ public class ScrewdriverApiTest extends ControllerContainerTest { tester.containerTester().updateSystemVersion(); Application app = tester.createApplication(); - try (Lock lock = tester.controller().applications().lock(app.id())) { - tester.controller().applications().store( - tester.controller().applications().require(app.id(), lock).withProjectId(1) - ); - } + tester.controller().applications().lockedOrThrow(app.id(), application -> + tester.controller().applications().store(application.withProjectId(1))); // Unknown application assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/foo/application/bar", @@ -163,7 +160,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + app.id().tenant().value() + "/application/" + app.id().application().value(), "invalid".getBytes(StandardCharsets.UTF_8), Request.Method.POST), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown job id 'invalid'\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown job name 'invalid'\"}"); // component is triggered if no job is specified in request body assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + @@ -172,7 +169,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { 200, "{\"message\":\"Triggered component for tenant1.application1\"}"); assertFalse(buildSystem.jobs().isEmpty()); - assertEquals(JobType.component.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.component.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(1L, buildSystem.jobs().get(0).projectId()); buildSystem.takeJobsToRun(); @@ -182,7 +179,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { "staging-test".getBytes(StandardCharsets.UTF_8), Request.Method.POST), 200, "{\"message\":\"Triggered staging-test for tenant1.application1\"}"); assertFalse(buildSystem.jobs().isEmpty()); - assertEquals(JobType.stagingTest.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.stagingTest.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(1L, buildSystem.jobs().get(0).projectId()); } @@ -197,14 +194,14 @@ public class ScrewdriverApiTest extends ControllerContainerTest { Optional<JobError> jobError) { return "{\n" + - " \"projectId\" : " + projectId + ",\n" + - " \"jobName\" :\"" + jobType.id() + "\",\n" + - " \"buildNumber\" : " + buildNumber + ",\n" + - jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + - " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + - " \"application\" :\"" + applicationId.application().value() + "\",\n" + - " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + - "}"; + " \"projectId\" : " + projectId + ",\n" + + " \"jobName\" :\"" + jobType.jobName() + "\",\n" + + " \"buildNumber\" : " + buildNumber + ",\n" + + jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + + " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + + " \"application\" :\"" + applicationId.application().value() + "\",\n" + + " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + + "}"; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json index e293d85b594..8ffd9511a96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Got notified about completion of job status of productionUsEast3[ last triggered: (never), last completed: (never), first failing: (not failing), lastSuccess: (never)], but that has not been triggered nor deployed" + "message": "Got notified about completion of job status of productionUsEast3[ last triggered: (never), last completed: (never), first failing: (not failing), lastSuccess: (never)], but that has neither been triggered nor deployed" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java new file mode 100644 index 00000000000..a00665b77cb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.zone.v1; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +/** + * @author mpolden + */ +public class ZoneApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/"; + private static final List<Zone> zones = Arrays.asList( + new Zone(Environment.prod, RegionName.from("us-north-1")), + new Zone(Environment.dev, RegionName.from("us-north-2")), + new Zone(Environment.test, RegionName.from("us-north-3")), + new Zone(Environment.staging, RegionName.from("us-north-4")) + ); + + private ContainerControllerTester tester; + + @Before + public void before() { + ZoneRegistryMock zoneRegistry = (ZoneRegistryMock) container.components() + .getComponent(ZoneRegistryMock.class.getName()); + zoneRegistry.setDefaultRegionForEnvironment(Environment.dev, RegionName.from("us-north-2")) + .setZones(zones); + this.tester = new ContainerControllerTester(container, responseFiles); + } + + @Test + public void test_requests() throws Exception { + // GET /zone/v1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1"), + new File("root.json")); + + // GET /zone/v1/environment/prod + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1/environment/prod"), + new File("prod.json")); + + // GET /zone/v1/environment/dev/default + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1/environment/dev/default"), + new File("default-for-region.json")); + } + + @Test + public void test_invalid_requests() throws Exception { + // GET /zone/v1/environment/prod/default: No default region + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1/environment/prod/default"), + new File("no-default-region.json"), + 400); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/default-for-region.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/default-for-region.json new file mode 100644 index 00000000000..7c4a7e2b4a5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/default-for-region.json @@ -0,0 +1,4 @@ +{ + "name": "us-north-2", + "url": "http://localhost:8080/zone/v2/environment/dev/region/us-north-2" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/no-default-region.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/no-default-region.json new file mode 100644 index 00000000000..bdc6601a2e9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/no-default-region.json @@ -0,0 +1,4 @@ +{ + "error-code": "BAD_REQUEST", + "message": "No default region for environment: prod" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/prod.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/prod.json new file mode 100644 index 00000000000..cebf48e6428 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/prod.json @@ -0,0 +1,6 @@ +[ + { + "name": "us-north-1", + "url": "http://localhost:8080/zone/v2/environment/prod/region/us-north-1" + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/root.json new file mode 100644 index 00000000000..b3bd5247414 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/root.json @@ -0,0 +1,18 @@ +[ + { + "name": "dev", + "url": "http://localhost:8080/zone/v2/environment/dev" + }, + { + "name": "prod", + "url": "http://localhost:8080/zone/v2/environment/prod" + }, + { + "name": "staging", + "url": "http://localhost:8080/zone/v2/environment/staging" + }, + { + "name": "test", + "url": "http://localhost:8080/zone/v2/environment/test" + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java new file mode 100644 index 00000000000..63899d808f9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -0,0 +1,117 @@ +package com.yahoo.vespa.hosted.controller.restapi.zone.v2; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Request.Method; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.hosted.controller.ConfigServerProxyMock; +import com.yahoo.vespa.hosted.controller.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author mpolden + */ +public class ZoneApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/"; + private static final List<Zone> zones = Arrays.asList( + new Zone(Environment.prod, RegionName.from("us-north-1")), + new Zone(Environment.dev, RegionName.from("us-north-2")), + new Zone(Environment.test, RegionName.from("us-north-3")), + new Zone(Environment.staging, RegionName.from("us-north-4")) + ); + + private ContainerControllerTester tester; + private ConfigServerProxyMock proxy; + + @Before + public void before() { + ZoneRegistryMock zoneRegistry = (ZoneRegistryMock) container.components() + .getComponent(ZoneRegistryMock.class.getName()); + zoneRegistry.setDefaultRegionForEnvironment(Environment.dev, RegionName.from("us-north-2")) + .setZones(zones); + this.tester = new ContainerControllerTester(container, responseFiles); + this.proxy = (ConfigServerProxyMock) container.components().getComponent(ConfigServerProxyMock.class.getName()); + } + + @Test + public void test_requests() throws Exception { + // GET /zone/v2 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2"), + new File("root.json")); + + // GET /zone/v2/prod/us-north-1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1"), + "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("GET", proxy.lastReceived().get().getMethod()); + + // GET /zone/v2/nodes/v2/node/?recursive=true + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/?recursive=true"), + "ok"); + + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/node/?recursive=true", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("GET", proxy.lastReceived().get().getMethod()); + + // POST /zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1", + new byte[0], Method.POST), + "ok"); + assertEquals("dev", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-2", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/command/restart?hostname=node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("POST", proxy.lastReceived().get().getMethod()); + + // PUT /zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1", + new byte[0], Method.PUT), "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/state/dirty/node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("PUT", proxy.lastReceived().get().getMethod()); + + // DELETE /zone/v2/prod/us-north-1/nodes/v2/node/node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", + new byte[0], Method.DELETE), "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("DELETE", proxy.lastReceived().get().getMethod()); + + // PATCH /zone/v2/prod/us-north-1/nodes/v2/node/node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Method.PATCH), "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("PATCH", proxy.lastReceived().get().getMethod()); + assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get()); + } + + @Test + public void test_invalid_requests() throws Exception { + // GET /zone/v2/prod/us-north-34/nodes/v2 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2", + new byte[0], Method.POST), + new File("unknown-zone.json"), 400); + assertFalse(proxy.lastReceived().isPresent()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json new file mode 100644 index 00000000000..ab168854267 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json @@ -0,0 +1,26 @@ +{ + "uris": [ + "http://localhost:8080/zone/v2/prod/us-north-1", + "http://localhost:8080/zone/v2/dev/us-north-2", + "http://localhost:8080/zone/v2/test/us-north-3", + "http://localhost:8080/zone/v2/staging/us-north-4" + ], + "zones": [ + { + "environment": "prod", + "region": "us-north-1" + }, + { + "environment": "dev", + "region": "us-north-2" + }, + { + "environment": "test", + "region": "us-north-3" + }, + { + "environment": "staging", + "region": "us-north-4" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/unknown-zone.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/unknown-zone.json new file mode 100644 index 00000000000..c7d6e4b8400 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/unknown-zone.json @@ -0,0 +1,4 @@ +{ + "error-code": "BAD_REQUEST", + "message": "No such zone: prod.us-north-42" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 519c457e73b..4f97c078c9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.Environment; @@ -19,6 +20,8 @@ import org.junit.Test; import java.net.URI; import java.net.URISyntaxException; +import java.time.Duration; +import java.util.Collections; import java.util.List; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; @@ -28,6 +31,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -95,23 +99,20 @@ public class VersionStatusTest { List<VespaVersion> versions = tester.controller().versionStatus().versions(); assertEquals("The two versions above exist", 2, versions.size()); + System.err.println(tester.controller().applications().deploymentTrigger().jobTimeoutLimit()); + VespaVersion v1 = versions.get(0); assertEquals(version1, v1.versionNumber()); - assertEquals(0, v1.statistics().failing().size()); - // All applications are on v1 in at least one zone - assertEquals(3, v1.statistics().production().size()); - assertTrue(v1.statistics().production().contains(app2.id())); - assertTrue(v1.statistics().production().contains(app1.id())); + assertEquals("No applications are failing on version1.", ImmutableSet.of(), v1.statistics().failing()); + assertEquals("All applications have at least one active production deployment on version 1.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v1.statistics().production()); + assertEquals("No applications have active deployment jobs on version1.", ImmutableSet.of(), v1.statistics().deploying()); VespaVersion v2 = versions.get(1); assertEquals(version2, v2.versionNumber()); - // All applications have failed on v2 in at least one zone - assertEquals(3, v2.statistics().failing().size()); - assertTrue(v2.statistics().failing().contains(app1.id())); - assertTrue(v2.statistics().failing().contains(app3.id())); - // Only one application is on v2 in at least one zone - assertEquals(1, v2.statistics().production().size()); - assertTrue(v2.statistics().production().contains(app2.id())); + assertEquals("All applications have failed on version2 in at least one zone.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().failing()); + assertEquals("Only app2 has successfully deployed to production on version2.", ImmutableSet.of(app2.id()), v2.statistics().production()); + // Should test the below, but can't easily be done with current test framework. This test passes in DeploymentApiTest. + // assertEquals("All applications are being retried on version2.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().deploying()); } @Test @@ -161,6 +162,12 @@ public class VersionStatusTest { assertEquals("One canary failed: Broken", Confidence.broken, confidence(tester.controller(), version1)); + // Finish running jobs + tester.deployAndNotify(canary2, DeploymentTester.applicationPackage("canary"), false, systemTest); + tester.clock().advance(Duration.ofHours(1)); + tester.deployAndNotify(canary1, DeploymentTester.applicationPackage("canary"), false, productionUsWest1); + tester.deployAndNotify(canary2, DeploymentTester.applicationPackage("canary"), false, systemTest); + // New version is released Version version2 = new Version("5.2"); tester.upgradeSystem(version2); @@ -204,6 +211,7 @@ public class VersionStatusTest { // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); + tester.completeUpgrade(default9, version2, "default"); tester.updateVersionStatus(); assertEquals("Confidence remains unchanged for version0: High", @@ -241,7 +249,7 @@ public class VersionStatusTest { } @Test - public void testIgnoreConfigdeince() { + public void testIgnoreConfidence() { DeploymentTester tester = new DeploymentTester(); Version version0 = new Version("5.0"); @@ -270,7 +278,6 @@ public class VersionStatusTest { tester.completeUpgradeWithError(default3, version1, "default", stagingTest); tester.completeUpgradeWithError(default4, version1, "default", stagingTest); tester.updateVersionStatus(); - assertEquals("Canaries have upgraded, 1 of 4 default apps failing: Broken", Confidence.broken, confidence(tester.controller(), version1)); @@ -295,8 +302,9 @@ public class VersionStatusTest { Version versionWithUnknownTag = new Version("6.1.2"); Application app = tester.createAndDeploy("tenant1", "domain1","application1", Environment.test, 11); - applications.notifyJobCompletion(mockReport(app, component, true)); - applications.notifyJobCompletion(mockReport(app, systemTest, true)); + tester.clock().advance(Duration.ofMillis(1)); + applications.notifyJobCompletion(DeploymentTester.jobReport(app, component, true)); + applications.notifyJobCompletion(DeploymentTester.jobReport(app, systemTest, true)); List<VespaVersion> vespaVersions = VersionStatus.compute(tester.controller()).versions(); @@ -313,14 +321,4 @@ public class VersionStatusTest { .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version)); } - private DeploymentJobs.JobReport mockReport(Application application, DeploymentJobs.JobType jobType, boolean success) { - return new DeploymentJobs.JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - JobError.from(success) - ); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java index 561799529f9..b4074fc1944 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.rotation; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; @@ -22,6 +22,10 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; /** * @author Oyvind Gronnesby @@ -100,12 +104,13 @@ public class ControllerRotationRepositoryTest { private ControllerRotationRepository repository; private ControllerRotationRepository repositoryWhitespaces; - + private Metric metric; @Before public void setup_repository() { - repository = new ControllerRotationRepository(rotationsConfig, controllerDb, MetricReceiver.nullImplementation); - repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, MetricReceiver.nullImplementation); + metric = mock(Metric.class); + repository = new ControllerRotationRepository(rotationsConfig, controllerDb, metric); + repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, metric); controllerDb.assignRotation(new RotationId("foo-1"), applicationId); } @@ -129,6 +134,7 @@ public class ControllerRotationRepositoryTest { Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpec); Rotation assignedRotation = new Rotation(new RotationId("foo-2"), "foo-2.com"); assertContainsOnly(assignedRotation, rotations); + verify(metric).set(eq(ControllerRotationRepository.REMAINING_ROTATIONS_METRIC_NAME), eq(1), any()); } @Test @@ -140,6 +146,7 @@ public class ControllerRotationRepositoryTest { thrown.expectMessage("no rotations available"); repository.getOrAssignRotation(third, deploymentSpec); + verify(metric).set(eq(ControllerRotationRepository.REMAINING_ROTATIONS_METRIC_NAME), eq(0), any()); } @Test diff --git a/dist/vespa.spec b/dist/vespa.spec index bf6e49fdf85..ca6a9504401 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -47,6 +47,14 @@ BuildRequires: boost-devel >= 1.60 BuildRequires: llvm-devel >= 4.0 BuildRequires: boost-devel >= 1.63 %endif +%if 0%{?fc27} +BuildRequires: llvm-devel >= 4.0 +BuildRequires: boost-devel >= 1.64 +%endif +%if 0%{?fc28} +BuildRequires: llvm-devel >= 4.0 +BuildRequires: boost-devel >= 1.64 +%endif BuildRequires: zookeeper-devel >= 3.4.9 %endif BuildRequires: lz4-devel @@ -65,6 +73,21 @@ Requires: epel-release %endif Requires: which Requires: initscripts +Requires: perl +Requires: perl-Carp +Requires: perl-Data-Dumper +Requires: perl-Digest-MD5 +Requires: perl-Env +Requires: perl-Exporter +Requires: perl-File-Path +Requires: perl-File-Temp +Requires: perl-Getopt-Long +Requires: perl-IO-Socket-IP +Requires: perl-JSON +Requires: perl-libwww-perl +Requires: perl-Net-INET6Glue +Requires: perl-Pod-Usage +Requires: perl-URI Requires: valgrind Requires: Judy Requires: lz4 @@ -80,14 +103,24 @@ Requires: vespa-zookeeper-c-client >= 3.4.9-6 %endif %if 0%{?fedora} %if 0%{?fc25} -Requires: llvm >= 3.9.1 +Requires: llvm-libs >= 3.9.1 Requires: boost >= 1.60 %endif %if 0%{?fc26} -Requires: llvm >= 4.0 +Requires: llvm-libs >= 4.0 Requires: boost >= 1.63 %define _vespa_llvm_version 4.0 %endif +%if 0%{?fc27} +Requires: llvm-libs >= 4.0 +Requires: boost >= 1.64 +%define _vespa_llvm_version 4.0 +%endif +%if 0%{?fc28} +Requires: llvm-libs >= 4.0 +Requires: boost >= 1.64 +%define _vespa_llvm_version 4.0 +%endif Requires: zookeeper >= 3.4.9 %define _extra_link_directory /opt/vespa-libtorrent/lib;/opt/vespa-cppunit/lib %define _extra_include_directory /opt/vespa-libtorrent/include;/opt/vespa-cppunit/include diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java index 7f47b638dde..a3805add15d 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Container.java @@ -9,6 +9,7 @@ import java.util.Objects; public class Container { public final String hostname; public final DockerImage image; + public final ContainerResources resources; public final ContainerName name; public final State state; public final int pid; @@ -16,11 +17,13 @@ public class Container { public Container( final String hostname, final DockerImage image, + final ContainerResources resources, final ContainerName containerName, final State state, final int pid) { this.hostname = hostname; this.image = image; + this.resources = resources; this.name = containerName; this.state = state; this.pid = pid; @@ -34,13 +37,14 @@ public class Container { final Container other = (Container) obj; return Objects.equals(hostname, other.hostname) && Objects.equals(image, other.image) + && Objects.equals(resources, other.resources) && Objects.equals(name, other.name) && Objects.equals(pid, other.pid); } @Override public int hashCode() { - return Objects.hash(hostname, image, name, pid); + return Objects.hash(hostname, image, resources, name, pid); } @Override @@ -48,6 +52,7 @@ public class Container { return "Container {" + " hostname=" + hostname + " image=" + image + + " resources=" + resources + " name=" + name + " state=" + state + " pid=" + pid diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java new file mode 100644 index 00000000000..aad0b07a2c4 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.dockerapi; + +/** + * @author valerijf + */ +public class ContainerResources { + public static final ContainerResources UNLIMITED = ContainerResources.from(0, 0); + + public final int cpuShares; + public final long memoryBytes; + + ContainerResources(int cpuShares, long memoryBytes) { + this.cpuShares = cpuShares; + this.memoryBytes = memoryBytes; + } + + public static ContainerResources from(double cpuCores, double memoryGb) { + return new ContainerResources( + (int) Math.round(10 * cpuCores), + (long) ((1L << 30) * memoryGb)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ContainerResources that = (ContainerResources) o; + + if (cpuShares != that.cpuShares) return false; + return memoryBytes == that.memoryBytes; + } + + @Override + public int hashCode() { + int result = cpuShares; + result = 31 * result + (int) (memoryBytes ^ (memoryBytes >>> 32)); + return result; + } + + @Override + public String toString() { + return cpuShares + " CPU Shares, " + memoryBytes + "B memory"; + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 8e8a650d906..485de99082b 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -25,6 +25,7 @@ import java.util.stream.IntStream; class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private final DockerClient docker; private final DockerImage dockerImage; + private final ContainerResources containerResources; private final ContainerName containerName; private final String hostName; private final Map<String, String> labels = new HashMap<>(); @@ -32,8 +33,6 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { private final List<String> volumeBindSpecs = new ArrayList<>(); private final List<Ulimit> ulimits = new ArrayList<>(); - private Optional<Long> memoryInB = Optional.empty(); - private Optional<Integer> cpuShares = Optional.empty(); private Optional<String> networkMode = Optional.empty(); private Optional<String> ipv4Address = Optional.empty(); private Optional<String> ipv6Address = Optional.empty(); @@ -43,10 +42,12 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { CreateContainerCommandImpl(DockerClient docker, DockerImage dockerImage, + ContainerResources containerResources, ContainerName containerName, String hostName) { this.docker = docker; this.dockerImage = dockerImage; + this.containerResources = containerResources; this.containerName = containerName; this.hostName = hostName; } @@ -103,18 +104,6 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { } @Override - public Docker.CreateContainerCommand withMemoryInMb(long megaBytes) { - memoryInB = Optional.of(megaBytes * 1024 * 1024); - return this; - } - - @Override - public Docker.CreateContainerCommand withCpuShares(int shares) { - cpuShares = Optional.of(shares); - return this; - } - - @Override public Docker.CreateContainerCommand withNetworkMode(String mode) { networkMode = Optional.of(mode); return this; @@ -144,6 +133,8 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { final CreateContainerCmd containerCmd = docker .createContainerCmd(dockerImage.asString()) + .withCpuShares(containerResources.cpuShares) + .withMemory(containerResources.memoryBytes) .withName(containerName.asString()) .withHostName(hostName) .withLabels(labels) @@ -157,8 +148,6 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { .filter(mode -> ! mode.toLowerCase().equals("host")) .ifPresent(mode -> containerCmd.withMacAddress(generateMACAddress(hostName, ipv4Address, ipv6Address))); - memoryInB.ifPresent(containerCmd::withMemory); - cpuShares.ifPresent(containerCmd::withCpuShares); networkMode.ifPresent(containerCmd::withNetworkMode); ipv4Address.ifPresent(containerCmd::withIpv4Address); ipv6Address.ifPresent(containerCmd::withIpv6Address); @@ -191,14 +180,14 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { return "--name " + containerName.asString() + " " + "--hostname " + hostName + " " + + "--cpu-shares " + containerResources.cpuShares + " " + + "--memory " + containerResources.memoryBytes + " " + toRepeatedOption("--label", labelList) + toRepeatedOption("--ulimit", ulimitList) + toRepeatedOption("--env", environmentAssignments) + toRepeatedOption("--volume", volumeBindSpecs) + toRepeatedOption("--cap-add", addCapabilitiesList) + toRepeatedOption("--cap-drop", dropCapabilitiesList) - + toOptionalOption("--memory", memoryInB) - + toOptionalOption("--cpu-shares", cpuShares) + toOptionalOption("--net", networkMode) + toOptionalOption("--ip", ipv4Address) + toOptionalOption("--ip6", ipv6Address) diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 865908bdc8e..2bf3f0f8d84 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -16,8 +16,6 @@ public interface Docker { CreateContainerCommand withLabel(String name, String value); CreateContainerCommand withEnvironment(String name, String value); CreateContainerCommand withVolume(String path, String volumePath); - CreateContainerCommand withMemoryInMb(long megaBytes); - CreateContainerCommand withCpuShares(int shares); CreateContainerCommand withNetworkMode(String mode); CreateContainerCommand withIpAddress(InetAddress address); CreateContainerCommand withUlimit(String name, int softLimit, int hardLimit); @@ -31,6 +29,7 @@ public interface Docker { CreateContainerCommand createContainerCommand( DockerImage dockerImage, + ContainerResources containerResources, ContainerName containerName, String hostName); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index a6f8783a22c..9de2cae604f 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -202,8 +202,9 @@ public class DockerImpl implements Docker { } @Override - public CreateContainerCommand createContainerCommand(DockerImage image, ContainerName name, String hostName) { - return new CreateContainerCommandImpl(dockerClient, image, name, hostName); + public CreateContainerCommand createContainerCommand(DockerImage image, ContainerResources containerResources, + ContainerName name, String hostName) { + return new CreateContainerCommandImpl(dockerClient, image, containerResources, name, hostName); } @Override @@ -369,6 +370,8 @@ public class DockerImpl implements Docker { new Container( response.getConfig().getHostName(), new DockerImage(response.getConfig().getImage()), + new ContainerResources(response.getHostConfig().getCpuShares(), + response.getHostConfig().getMemory()), new ContainerName(decode(response.getName())), Container.State.valueOf(response.getState().getStatus().toUpperCase()), response.getState().getPid() diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java index 310fb4ffdd3..18f87e5ae17 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java @@ -41,18 +41,18 @@ public class DockerTest { InetAddress inetAddress1 = InetAddress.getByName("172.18.10.10"); InetAddress inetAddress2 = InetAddress.getByName("172.18.10.11"); - docker.createContainerCommand(dockerImage, containerName1, hostName1) + docker.createContainerCommand(dockerImage, ContainerResources.from(0, 0.1), containerName1, hostName1) .withManagedBy(MANAGER_NAME) .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) .withIpAddress(inetAddress1) - .withMemoryInMb(100).create(); + .create(); docker.startContainer(containerName1); - docker.createContainerCommand(dockerImage, containerName2, hostName2) + docker.createContainerCommand(dockerImage, ContainerResources.from(0, 0.1), containerName2, hostName2) .withManagedBy(MANAGER_NAME) .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) .withIpAddress(inetAddress2) - .withMemoryInMb(100).create(); + .create(); docker.startContainer(containerName2); // 137 = 128 + 9 = kill -9 (SIGKILL), doesn't need to be run as "root", but "yahoo" does not exist in this basic image @@ -74,7 +74,8 @@ public class DockerTest { final ContainerName containerName = new ContainerName("docker-test-foo"); final String containerHostname = "hostName1"; - docker.createContainerCommand(dockerImage, containerName, containerHostname).withManagedBy(MANAGER_NAME).create(); + docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName, containerHostname) + .withManagedBy(MANAGER_NAME).create(); Optional<Container> container = docker.getContainer(containerName); assertTrue(container.isPresent()); assertEquals(container.get().state, Container.State.CREATED); @@ -110,7 +111,8 @@ public class DockerTest { final ContainerName containerName = new ContainerName("docker-test-foo"); final String containerHostname = "hostName1"; - docker.createContainerCommand(dockerImage, containerName, containerHostname).withManagedBy(MANAGER_NAME).create(); + docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName, containerHostname) + .withManagedBy(MANAGER_NAME).create(); docker.startContainer(containerName); docker.executeInContainerAsRoot(containerName, 1L, "sh", "-c", "sleep 5"); } @@ -128,7 +130,8 @@ public class DockerTest { final ContainerName containerName = new ContainerName("docker-test-foo"); final String containerHostname = "hostName1"; - docker.createContainerCommand(dockerImage, containerName, containerHostname).withManagedBy(MANAGER_NAME).create(); + docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName, containerHostname) + .withManagedBy(MANAGER_NAME).create(); docker.startContainer(containerName); docker.executeInContainerAsRoot(containerName, 2L, "sh", "-c", "echo hei"); @@ -145,11 +148,13 @@ public class DockerTest { InetAddress inetAddress1 = InetAddress.getByName("172.18.10.10"); InetAddress inetAddress2 = InetAddress.getByName("172.18.10.11"); - docker.createContainerCommand(dockerImage, containerName1, hostName1).withManagedBy(MANAGER_NAME) + docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName1, hostName1) + .withManagedBy(MANAGER_NAME) .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withIpAddress(inetAddress1).create(); docker.startContainer(containerName1); - docker.createContainerCommand(dockerImage, containerName2, hostName2).withManagedBy(MANAGER_NAME) + docker.createContainerCommand(dockerImage, ContainerResources.UNLIMITED, containerName2, hostName2) + .withManagedBy(MANAGER_NAME) .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withIpAddress(inetAddress2).create(); docker.startContainer(containerName2); diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/RunSystemTests.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/RunSystemTests.java index 9613470a735..715c839e531 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/RunSystemTests.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/RunSystemTests.java @@ -126,6 +126,7 @@ public class RunSystemTests { InetAddress nodeInetAddress = InetAddress.getByName(containerName.asString()); docker.createContainerCommand( SYSTEMTESTS_DOCKER_IMAGE, + ContainerResources.UNLIMITED, containerName, containerName.asString()) .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) diff --git a/document/src/main/java/com/yahoo/document/select/ResultList.java b/document/src/main/java/com/yahoo/document/select/ResultList.java index 41f0fb5edef..8d73c475981 100644 --- a/document/src/main/java/com/yahoo/document/select/ResultList.java +++ b/document/src/main/java/com/yahoo/document/select/ResultList.java @@ -60,6 +60,10 @@ public class ResultList { return results; } + public static ResultList fromBoolean(boolean result) { + return new ResultList(result ? Result.TRUE : Result.FALSE); + } + public Result toResult() { if (results.isEmpty()) { return Result.FALSE; diff --git a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java index 34bcf914d17..a54f5cada97 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java @@ -50,7 +50,7 @@ public class ArithmeticNode implements ExpressionNode { Object val = item.node.evaluate(context); if (val == null) { - throw new IllegalStateException("Null value found!"); + throw new IllegalArgumentException("Can not perform arithmetic on null value (referencing missing field?)"); } if (val instanceof AttributeNode.VariableValueList) { diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java index b8281b9ed0a..bbd244dd1dc 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -127,12 +127,14 @@ public class AttributeNode implements ExpressionNode { FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth); IteratorHandler handler = new IteratorHandler(); doc.iterateNested(fieldPath, 0, handler); + if (handler.values.isEmpty()) { + return null; + } return handler.values; } else if (value instanceof DocumentUpdate) { return Result.INVALID; } return Result.FALSE; - //throw new IllegalStateException("Attributes are only available for document types for value '" + value + "'. Looking for " + fieldPth); } private static Object evaluateFunction(String function, Object value) { diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index a13641adadf..372b61bb493 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -215,8 +215,8 @@ public class ComparisonNode implements ExpressionNode { public Object evaluate(Context context) { Object oLeft = lhs.evaluate(context); Object oRight = rhs.evaluate(context); - if (oLeft == null && oRight == null) { - return new ResultList(Result.TRUE); + if (oLeft == null || oRight == null) { + return evaluateWithAtLeastOneNullSide(oLeft, oRight); } if (oLeft == Result.INVALID || oRight == Result.INVALID) { return new ResultList(Result.INVALID); @@ -237,6 +237,23 @@ public class ComparisonNode implements ExpressionNode { return new ResultList(evaluateBool(oLeft, oRight)); } + /** + * Evaluates a binary comparison where one or both operands are null. + * Boolean outcomes are only defined for (in)equality relations, all others + * return Result.INVALID. + * + * Precondition: lhs AND/OR rhs is null. + */ + private ResultList evaluateWithAtLeastOneNullSide(Object lhs, Object rhs) { + if (operator.equals("==") || operator.equals("=")) { // Glob (=) operator falls back to equality for non-strings + return ResultList.fromBoolean(lhs == rhs); + } else if (operator.equals("!=")) { + return ResultList.fromBoolean(lhs != rhs); + } else { + return new ResultList(Result.INVALID); + } + } + public ResultList evaluateListsTrue(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) { if (lhs.size() != rhs.size()) { return new ResultList(Result.FALSE); diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 7b6130185b4..508bf7f0b18 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -263,7 +263,7 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { assertEquals(Result.FALSE, evaluate("test.content = 1 and true", put)); // BROKEN assertEquals(Result.INVALID, evaluate("test.content = 1 and true", upd)); - assertEquals(Result.FALSE, evaluate("test.content = 1 or true", put)); // BROKEN + assertEquals(Result.TRUE, evaluate("test.content = 1 or true", put)); assertEquals(Result.TRUE, evaluate("test.content = 1 or true", upd)); assertEquals(Result.FALSE, evaluate("test.content = 1 and false", put)); @@ -275,7 +275,7 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { assertEquals(Result.FALSE, evaluate("true and test.content = 1", put)); // BROKEN assertEquals(Result.INVALID, evaluate("true and test.content = 1", upd)); - assertEquals(Result.FALSE, evaluate("true or test.content = 1", put)); // BROKEN + assertEquals(Result.TRUE, evaluate("true or test.content = 1", put)); assertEquals(Result.TRUE, evaluate("true or test.content = 1", upd)); assertEquals(Result.FALSE, evaluate("false and test.content = 1", put)); @@ -420,8 +420,10 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { assertEquals(Result.TRUE, evaluate("30 != \"foo\"", documents.get(0))); assertEquals(Result.INVALID, evaluate("14.2 <= \"foo\"", documents.get(0))); assertEquals(Result.TRUE, evaluate("null == null", documents.get(0))); - assertEquals(Result.TRUE, evaluate("null = null", documents.get(0))); + assertEquals(Result.TRUE, evaluate("null = null", documents.get(0))); // Glob operator falls back to == comparison + assertEquals(Result.FALSE, evaluate("null != null", documents.get(0))); assertEquals(Result.FALSE, evaluate("\"bar\" == null", documents.get(0))); + assertEquals(Result.FALSE, evaluate("null == \"bar\"", documents.get(0))); assertEquals(Result.FALSE, evaluate("14.3 == null", documents.get(0))); assertEquals(Result.FALSE, evaluate("null = 0", documents.get(0))); @@ -441,6 +443,20 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { assertEquals(Result.FALSE, evaluate("test.hstring == test.content", documents.get(0))); assertEquals(Result.TRUE, evaluate("test.hstring == test.content", documents.get(2))); assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", documents.get(1))); + // Case where field is not present (i.e. null) is defined for (in)equality comparisons, but + // not for other relations. + assertEquals(Result.TRUE, evaluate("test.hint != 1234", documents.get(7))); + assertEquals(Result.FALSE, evaluate("test.hint == 1234", documents.get(7))); + assertEquals(Result.INVALID, evaluate("test.hint < 1234", documents.get(7))); + // Propagation of Invalid through logical operators should match C++ implementation + assertEquals(Result.FALSE, evaluate("test.hint < 1234 and false", documents.get(7))); + assertEquals(Result.INVALID, evaluate("test.hint < 1234 and true", documents.get(7))); + assertEquals(Result.TRUE, evaluate("test.hint < 1234 or true", documents.get(7))); + assertEquals(Result.INVALID, evaluate("test.hint < 1234 or false", documents.get(7))); + // Must be possible to predicate a sub-expression on the presence of a field without + // propagating up an Invalid value from the comparison. + assertEquals(Result.FALSE, evaluate("test.hint and test.hint < 1234", documents.get(7))); + assertEquals(Result.FALSE, evaluate("test.hint != null and test.hint < 1234", documents.get(7))); // Document types. assertEquals(Result.TRUE, evaluate("test", documents.get(0))); @@ -457,6 +473,19 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { assertEquals(Result.TRUE, evaluate("not test.hint", documents.get(7))); assertEquals(Result.TRUE, evaluate("not test.hstring", documents.get(7))); + assertEquals(Result.TRUE, evaluate("test.hint != null", documents.get(0))); + assertEquals(Result.TRUE, evaluate("null != test.hint", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.hint == null", documents.get(0))); + assertEquals(Result.FALSE, evaluate("null == test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("null == test.hint", documents.get(7))); + assertEquals(Result.TRUE, evaluate("test.hint == null", documents.get(7))); + assertEquals(Result.FALSE, evaluate("test.hint != null", documents.get(7))); + assertEquals(Result.FALSE, evaluate("null != test.hint", documents.get(7))); + + assertEquals(Result.TRUE, evaluate("test.hint or true", documents.get(7))); + assertEquals(Result.TRUE, evaluate("not test.hint and true", documents.get(7))); + assertEquals(Result.FALSE, evaluate("not test.hint and false", documents.get(7))); + // Id values. assertEquals(Result.TRUE, evaluate("id == \"doc:myspace:anything\"", documents.get(0))); assertEquals(Result.TRUE, evaluate(" iD== \"doc:myspace:anything\" ", documents.get(0))); @@ -548,7 +577,7 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { assertEquals(Result.TRUE, evaluate("test.mystruct == test.mystruct", documents.get(1))); assertEquals(Result.FALSE, evaluate("test.mystruct != test.mystruct", documents.get(0))); assertEquals(Result.FALSE, evaluate("test.mystruct != test.mystruct", documents.get(1))); - //assertEquals(Result.INVALID, evaluate("test.mystruct < test.mystruct", documents.get(0))); + assertEquals(Result.INVALID, evaluate("test.mystruct < test.mystruct", documents.get(0))); //assertEquals(Result.FALSE, evaluate("test.mystruct < test.mystruct", documents.get(1))); //assertEquals(Result.INVALID, evaluate("test.mystruct < 5", documents.get(1))); //assertEquals(Result.INVALID, evaluate("test.mystruct == \"foo\"", documents.get(1))); @@ -584,7 +613,7 @@ public class DocumentSelectorTestCase extends junit.framework.TestCase { // Globbing/regexp of struct fields assertEquals(Result.FALSE, evaluate("test.mystruct.value = \"struc?val\"", documents.get(0))); assertEquals(Result.TRUE, evaluate("test.mystruct.value = \"struc?val\"", documents.get(1))); - assertEquals(Result.FALSE, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(0))); + assertEquals(Result.INVALID, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(0))); // Invalid since lhs is null assertEquals(Result.TRUE, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(1))); assertEquals(Result.FALSE, evaluate("test.intarray < 5", documents.get(0))); diff --git a/document/src/vespa/document/bucket/bucket.cpp b/document/src/vespa/document/bucket/bucket.cpp index d2855c0b3a1..b79d45eee34 100644 --- a/document/src/vespa/document/bucket/bucket.cpp +++ b/document/src/vespa/document/bucket/bucket.cpp @@ -23,4 +23,9 @@ vespalib::asciistream& operator<<(vespalib::asciistream& os, const Bucket& id) return os << "Bucket(" << id.getBucketSpace() << ", " << id.getBucketId() << ")"; } +std::ostream& operator<<(std::ostream& os, const Bucket& id) +{ + return os << id.toString(); +} + } diff --git a/document/src/vespa/document/bucket/bucket.h b/document/src/vespa/document/bucket/bucket.h index 5aa9b360c42..44068e1c443 100644 --- a/document/src/vespa/document/bucket/bucket.h +++ b/document/src/vespa/document/bucket/bucket.h @@ -45,5 +45,6 @@ private: }; vespalib::asciistream& operator<<(vespalib::asciistream&, const Bucket&); +std::ostream& operator<<(std::ostream&, const Bucket&); } diff --git a/document/src/vespa/document/bucket/bucketspace.cpp b/document/src/vespa/document/bucket/bucketspace.cpp index 304631ce334..7c1c816b5ba 100644 --- a/document/src/vespa/document/bucket/bucketspace.cpp +++ b/document/src/vespa/document/bucket/bucketspace.cpp @@ -20,4 +20,9 @@ vespalib::asciistream& operator<<(vespalib::asciistream& os, const BucketSpace& << ")"; } +std::ostream& operator<<(std::ostream& os, const BucketSpace& bucketSpace) +{ + return os << bucketSpace.toString(); +} + } diff --git a/document/src/vespa/document/bucket/bucketspace.h b/document/src/vespa/document/bucket/bucketspace.h index c13c81dbd73..1198b173a4b 100644 --- a/document/src/vespa/document/bucket/bucketspace.h +++ b/document/src/vespa/document/bucket/bucketspace.h @@ -42,5 +42,6 @@ private: }; vespalib::asciistream& operator<<(vespalib::asciistream&, const BucketSpace&); +std::ostream& operator<<(std::ostream&, const BucketSpace&); } diff --git a/document/src/vespa/document/test/make_bucket_space.cpp b/document/src/vespa/document/test/make_bucket_space.cpp index a213e5e36b7..be8292fcf71 100644 --- a/document/src/vespa/document/test/make_bucket_space.cpp +++ b/document/src/vespa/document/test/make_bucket_space.cpp @@ -9,4 +9,17 @@ BucketSpace makeBucketSpace() return BucketSpace::placeHolder(); } +BucketSpace makeBucketSpace(const vespalib::string &docTypeName) +{ + // Used by persistence conformance test to map fron document type name + // to bucket space. See document::TestDocRepo for known document types. + if (docTypeName == "no") { + return BucketSpace(2); + } else if (docTypeName == "testdoctype2") { + return BucketSpace(1); + } else { + return makeBucketSpace(); + } +} + } diff --git a/document/src/vespa/document/test/make_bucket_space.h b/document/src/vespa/document/test/make_bucket_space.h index 8289ec2030a..8b17eaea1ac 100644 --- a/document/src/vespa/document/test/make_bucket_space.h +++ b/document/src/vespa/document/test/make_bucket_space.h @@ -6,8 +6,9 @@ namespace document::test { -// Helper function used by unit tests +// Helper functions used by unit tests BucketSpace makeBucketSpace(); +BucketSpace makeBucketSpace(const vespalib::string &docTypeName); } diff --git a/document/src/vespa/document/test/make_document_bucket.cpp b/document/src/vespa/document/test/make_document_bucket.cpp index f4d22f7b83f..a24f757c99f 100644 --- a/document/src/vespa/document/test/make_document_bucket.cpp +++ b/document/src/vespa/document/test/make_document_bucket.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "make_document_bucket.h" +#include "make_bucket_space.h" namespace document::test { Bucket makeDocumentBucket(BucketId bucketId) { - return Bucket(BucketSpace::placeHolder(), bucketId); + return Bucket(makeBucketSpace(), bucketId); } } diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp index aa3fba01ef9..077ea52e255 100644 --- a/documentapi/src/tests/messagebus/messagebus_test.cpp +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/document/base/testdocrepo.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/documentapi.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/documentapi/loadtypes/loadtypeset.h> using document::DocumentTypeRepo; using document::readDocumenttypesConfig; @@ -14,7 +14,6 @@ using namespace documentapi; using mbus::Blob; using mbus::Routable; using mbus::IRoutingPolicy; -using document::test::makeDocumentBucket; class Test : public vespalib::TestApp { DocumentTypeRepo::SP _repo; @@ -110,11 +109,11 @@ void Test::get_document_message_is_not_sequenced() { } void Test::stat_bucket_message_is_not_sequenced() { - StatBucketMessage message(makeDocumentBucket(document::BucketId(16, 1)), ""); + StatBucketMessage message(document::BucketId(16, 1), ""); EXPECT_FALSE(message.hasSequenceId()); } void Test::get_bucket_list_message_is_not_sequenced() { - GetBucketListMessage message(makeDocumentBucket(document::BucketId(16, 1))); + GetBucketListMessage message(document::BucketId(16, 1)); EXPECT_FALSE(message.hasSequenceId()); } diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index d4052f30958..b50f30c1f91 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -1,17 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages50test.h" -#include <vespa/documentapi/documentapi.h> -#include <vespa/vdslib/container/writabledocumentlist.h> -#include <vespa/document/update/fieldpathupdates.h> -#include <vespa/document/datatype/documenttype.h> #include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/select/parser.h> -#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vdslib/container/writabledocumentlist.h> using document::DataType; using document::DocumentTypeRepo; -using document::test::makeDocumentBucket; /////////////////////////////////////////////////////////////////////////////// // @@ -90,7 +88,7 @@ createDoc(const DocumentTypeRepo &repo, const string &type_name, const string &i bool Messages50Test::testGetBucketListMessage() { - GetBucketListMessage msg(makeDocumentBucket(document::BucketId(16, 123))); + GetBucketListMessage msg(document::BucketId(16, 123)); msg.setLoadType(_loadTypes["foo"]); EXPECT_EQUAL(string("foo"), msg.getLoadType().getName()); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u, serialize("GetBucketListMessage", msg)); @@ -100,7 +98,7 @@ Messages50Test::testGetBucketListMessage() if (EXPECT_TRUE(obj.get() != NULL)) { GetBucketListMessage &ref = static_cast<GetBucketListMessage&>(*obj); EXPECT_EQUAL(string("foo"), ref.getLoadType().getName()); - EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucket().getBucketId()); + EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); } } return true; @@ -134,7 +132,7 @@ Messages50Test::testEmptyBucketsMessage() bool Messages50Test::testStatBucketMessage() { - StatBucketMessage msg(makeDocumentBucket(document::BucketId(16, 123)), "id.user=123"); + StatBucketMessage msg(document::BucketId(16, 123), "id.user=123"); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 27u, serialize("StatBucketMessage", msg)); @@ -142,7 +140,7 @@ Messages50Test::testStatBucketMessage() mbus::Routable::UP obj = deserialize("StatBucketMessage", DocumentProtocol::MESSAGE_STATBUCKET, lang); if (EXPECT_TRUE(obj.get() != NULL)) { StatBucketMessage &ref = static_cast<StatBucketMessage&>(*obj); - EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucket().getBucketId()); + EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); EXPECT_EQUAL("id.user=123", ref.getDocumentSelection()); } } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp index 49e8e048db5..96408e9a204 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp @@ -6,11 +6,11 @@ namespace documentapi { -GetBucketListMessage::GetBucketListMessage(const document::Bucket &bucket) : +GetBucketListMessage::GetBucketListMessage(const document::BucketId &bucketId) : DocumentMessage(), - _bucket(bucket) + _bucketId(bucketId), + _bucketSpace() { - // empty } DocumentReply::UP diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h index 6b70808aea5..8b49afd6672 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h @@ -2,13 +2,15 @@ #pragma once #include "documentmessage.h" -#include <vespa/document/bucket/bucket.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/vespalib/stllike/string.h> namespace documentapi { class GetBucketListMessage : public DocumentMessage { private: - document::Bucket _bucket; + document::BucketId _bucketId; + vespalib::string _bucketSpace; protected: // Implements DocumentMessage. @@ -18,17 +20,19 @@ public: /** * Constructs a new message with initial content. * - * @param bucket The bucket whose list to retrieve. + * @param bucketId The bucket whose list to retrieve. */ - GetBucketListMessage(const document::Bucket &bucket); + GetBucketListMessage(const document::BucketId &bucketId); /** * Returns the bucket whose list to retrieve. * * @return The bucket. */ - const document::Bucket &getBucket() const { return _bucket; } + const document::BucketId &getBucketId() const { return _bucketId; } + const vespalib::string &getBucketSpace() const { return _bucketSpace; } + void setBucketSpace(const vespalib::string &value) { _bucketSpace = value; } uint32_t getType() const override; string toString() const override { return "getbucketlistmessage"; } }; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp index 7977f13902c..a8ecef79fae 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp @@ -12,7 +12,9 @@ RemoveLocationMessage::RemoveLocationMessage( const document::BucketIdFactory& factory, document::select::Parser& parser, const string& documentSelection) - : _documentSelection(documentSelection) + : _documentSelection(documentSelection), + _bucketId(), + _bucketSpace() { document::BucketSelector bucketSel(factory); std::unique_ptr<document::BucketSelector::BucketVector> exprResult( diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h index a08a553b829..f58f115a8ec 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h @@ -20,6 +20,8 @@ public: const string& getDocumentSelection() const { return _documentSelection; } const document::BucketId& getBucketId() const { return _bucketId; }; + const string &getBucketSpace() const { return _bucketSpace; } + void setBucketSpace(const string &value) { _bucketSpace = value; } uint32_t getType() const override; string toString() const override { return "removelocationmessage"; } protected: @@ -28,6 +30,7 @@ protected: private: string _documentSelection; document::BucketId _bucketId; + string _bucketSpace; }; } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp index 00aef46cb79..213cc7d9932 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp @@ -3,20 +3,20 @@ #include "statbucketreply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -using document::BucketSpace; - namespace documentapi { StatBucketMessage::StatBucketMessage() : DocumentMessage(), - _bucket(BucketSpace::placeHolder(), document::BucketId()), - _documentSelection() + _bucketId(document::BucketId()), + _documentSelection(), + _bucketSpace() {} -StatBucketMessage::StatBucketMessage(document::Bucket bucket, const string& documentSelection) : +StatBucketMessage::StatBucketMessage(document::BucketId bucketId, const string& documentSelection) : DocumentMessage(), - _bucket(bucket), - _documentSelection(documentSelection) + _bucketId(bucketId), + _documentSelection(documentSelection), + _bucketSpace() {} StatBucketMessage::~StatBucketMessage() { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h index a13d675ee0b..5233205da4a 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h @@ -2,14 +2,15 @@ #pragma once #include "documentmessage.h" -#include <vespa/document/bucket/bucket.h> +#include <vespa/document/bucket/bucketid.h> namespace documentapi { class StatBucketMessage : public DocumentMessage { private: - document::Bucket _bucket; - string _documentSelection; + document::BucketId _bucketId; + string _documentSelection; + string _bucketSpace; protected: DocumentReply::UP doCreateReply() const override; @@ -23,9 +24,9 @@ public: /** * Constructs a new message with initial content. * - * @param bucket The bucket whose list to retrieve. + * @param bucketId The bucket whose list to retrieve. */ - StatBucketMessage(document::Bucket bucket, const string& documentSelection); + StatBucketMessage(document::BucketId bucket, const string& documentSelection); ~StatBucketMessage(); @@ -34,14 +35,14 @@ public: * * @return The bucket id. */ - document::Bucket getBucket() const { return _bucket; } + document::BucketId getBucketId() const { return _bucketId; } /** * Set the bucket to stat. * - * @param id The identifier to set. + * @param bucketId The identifier to set. */ - void setBucket(document::Bucket bucket) { _bucket = bucket; }; + void setBucketId(document::BucketId bucketId) { _bucketId = bucketId; }; /** * Returns the document selection used to filter the documents @@ -57,6 +58,9 @@ public: * @param value The selection string to set. */ void setDocumentSelection(const string &value) { _documentSelection = value; }; + + const string &getBucketSpace() const { return _bucketSpace; } + void setBucketSpace(const string &value) { _bucketSpace = value; } uint32_t getType() const override; string toString() const override { return "statbucketmessage"; } }; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp index b3e6ac422eb..c870775fae7 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -7,6 +7,11 @@ namespace documentapi { CreateVisitorMessage::CreateVisitorMessage() : DocumentMessage(), + _libName(), + _instanceId(), + _controlDestination(), + _dataDestination(), + _bucketSpace(), _docSelection(), _maxPendingReplyCount(8), _buckets(), @@ -30,6 +35,7 @@ CreateVisitorMessage::CreateVisitorMessage(const string& libraryName, _instanceId(instanceId), _controlDestination(controlDestination), _dataDestination(dataDestination), + _bucketSpace(), _docSelection(), _maxPendingReplyCount(8), _buckets(), diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index 7b60765b9c1..4ba6b1b437b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -28,6 +28,7 @@ private: string _instanceId; string _controlDestination; string _dataDestination; + string _bucketSpace; string _docSelection; uint32_t _maxPendingReplyCount; std::vector<document::BucketId> _buckets; @@ -69,6 +70,9 @@ public: const string& getDataDestination() const { return _dataDestination; } void setDataDestination(const string& value) { _dataDestination = value; } + const string& getBucketSpace() const { return _bucketSpace; } + void setBucketSpace(const string& value) { _bucketSpace = value; } + const vdslib::Parameters& getParameters() const { return _params; } vdslib::Parameters& getParameters() { return _params; } void setParameters(const vdslib::Parameters& params) { _params = params; } diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index b638fd1cabe..b7b451e8ddf 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -139,11 +139,11 @@ StoragePolicy::doSelect(mbus::RoutingContext &context) break; case DocumentProtocol::MESSAGE_STATBUCKET: - id = static_cast<const StatBucketMessage&>(msg).getBucket().getBucketId(); + id = static_cast<const StatBucketMessage&>(msg).getBucketId(); break; case DocumentProtocol::MESSAGE_GETBUCKETLIST: - id = static_cast<const GetBucketListMessage&>(msg).getBucket().getBucketId(); + id = static_cast<const GetBucketListMessage&>(msg).getBucketId(); break; case DocumentProtocol::MESSAGE_CREATEVISITOR: diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index 6876b4c3a71..4a022d8cabd 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -10,7 +10,6 @@ using vespalib::nbostream; using std::make_unique; using std::make_shared; -using document::BucketSpace; namespace documentapi { @@ -409,15 +408,14 @@ DocumentMessage::UP RoutableFactories50::GetBucketListMessageFactory::doDecode(document::ByteBuffer &buf) const { document::BucketId bucketId(decodeLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); - return std::make_unique<GetBucketListMessage>(bucket); + return std::make_unique<GetBucketListMessage>(bucketId); } bool RoutableFactories50::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); - buf.putLong(msg.getBucket().getBucketId().getRawId()); + buf.putLong(msg.getBucketId().getRawId()); return true; } @@ -837,8 +835,7 @@ RoutableFactories50::StatBucketMessageFactory::doDecode(document::ByteBuffer &bu DocumentMessage::UP ret(new StatBucketMessage()); StatBucketMessage &msg = static_cast<StatBucketMessage&>(*ret); - document::Bucket bucket(BucketSpace::placeHolder(), document::BucketId(decodeLong(buf))); - msg.setBucket(bucket); + msg.setBucketId(document::BucketId(decodeLong(buf))); msg.setDocumentSelection(decodeString(buf)); return ret; @@ -849,7 +846,7 @@ RoutableFactories50::StatBucketMessageFactory::doEncode(const DocumentMessage &o { const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); - buf.putLong(msg.getBucket().getBucketId().getRawId()); + buf.putLong(msg.getBucketId().getRawId()); buf.putString(msg.getDocumentSelection()); return true; diff --git a/eval/src/apps/eval_expr/eval_expr.cpp b/eval/src/apps/eval_expr/eval_expr.cpp index 2e1f7f7fdcb..71c808174b8 100644 --- a/eval/src/apps/eval_expr/eval_expr.cpp +++ b/eval/src/apps/eval_expr/eval_expr.cpp @@ -26,7 +26,7 @@ int main(int argc, char **argv) { if (result.is_double()) { fprintf(stdout, "%.32g\n", result.as_double()); } else if (result.is_tensor()) { - vespalib::string str = SimpleTensorEngine::ref().to_spec(*result.as_tensor()).to_string(); + vespalib::string str = SimpleTensorEngine::ref().to_spec(result).to_string(); fprintf(stdout, "%s\n", str.c_str()); } else { fprintf(stdout, "error\n"); diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 993d226c3c6..0aba5276ace 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -93,6 +93,7 @@ void generate_tensor_map(TestBuilder &dst) { generate_op1_map("isNan(a)", operation::IsNan::f, Mask2Seq(SkipNth(3), 1.0, my_nan), dst); generate_op1_map("relu(a)", operation::Relu::f, Sub2(Div10(N())), dst); generate_op1_map("sigmoid(a)", operation::Sigmoid::f, Sub2(Div10(N())), dst); + generate_op1_map("elu(a)", operation::Elu::f, Sub2(Div10(N())), dst); generate_op1_map("a in [1,5,7,13,42]", MyIn::f, N(), dst); generate_map_expr("map(a,f(a)((a+1)*2))", MyOp::f, Div10(N()), dst); } diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index d1163fb579d..616b98f0809 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -60,69 +60,42 @@ nbostream extract_data(const Inspector &value) { //----------------------------------------------------------------------------- -TensorSpec to_spec(const Value &value) { - if (value.is_error()) { - return TensorSpec("error"); - } else if (value.is_double()) { - return TensorSpec("double").add({}, value.as_double()); - } else { - ASSERT_TRUE(value.is_tensor()); - auto tensor = value.as_tensor(); - return tensor->engine().to_spec(*tensor); - } -} - -const Value &to_value(const TensorSpec &spec, const TensorEngine &engine, Stash &stash) { - if (spec.type() == "error") { - return stash.create<ErrorValue>(); - } else if (spec.type() == "double") { - double value = 0.0; - for (const auto &cell: spec.cells()) { - value += cell.second; - } - return stash.create<DoubleValue>(value); - } else { - ASSERT_TRUE(starts_with(spec.type(), "tensor(")); - return stash.create<TensorValue>(engine.create(spec)); - } -} - void insert_value(Cursor &cursor, const vespalib::string &name, const TensorSpec &spec) { - Stash stash; nbostream data; - const Value &value = to_value(spec, SimpleTensorEngine::ref(), stash); - SimpleTensorEngine::ref().encode(value, data, stash); + Value::UP value = SimpleTensorEngine::ref().from_spec(spec); + SimpleTensorEngine::ref().encode(*value, data); cursor.setData(name, Memory(data.peek(), data.size())); } TensorSpec extract_value(const Inspector &inspector) { - Stash stash; nbostream data = extract_data(inspector); - return to_spec(SimpleTensorEngine::ref().decode(data, stash)); + const auto &engine = SimpleTensorEngine::ref(); + return engine.to_spec(*engine.decode(data)); } //----------------------------------------------------------------------------- -std::vector<ValueType> get_types(const std::vector<Value::CREF> ¶m_values) { +std::vector<ValueType> get_types(const std::vector<Value::UP> ¶m_values) { std::vector<ValueType> param_types; for (size_t i = 0; i < param_values.size(); ++i) { - param_types.emplace_back(param_values[i].get().type()); + param_types.emplace_back(param_values[i]->type()); } return param_types; } TensorSpec eval_expr(const Inspector &test, const TensorEngine &engine, bool typed) { - Stash stash; Function fun = Function::parse(test["expression"].asString().make_string()); - std::vector<Value::CREF> param_values; + std::vector<Value::UP> param_values; + std::vector<Value::CREF> param_refs; for (size_t i = 0; i < fun.num_params(); ++i) { - param_values.emplace_back(to_value(extract_value(test["inputs"][fun.param_name(i)]), engine, stash)); + param_values.emplace_back(engine.from_spec(extract_value(test["inputs"][fun.param_name(i)]))); + param_refs.emplace_back(*param_values.back()); } NodeTypes types = typed ? NodeTypes(fun, get_types(param_values)) : NodeTypes(); InterpretedFunction ifun(engine, fun, types); InterpretedFunction::Context ctx(ifun); - InterpretedFunction::SimpleObjectParams params(param_values); - return to_spec(ifun.eval(ctx, params)); + InterpretedFunction::SimpleObjectParams params(param_refs); + return engine.to_spec(ifun.eval(ctx, params)); } //----------------------------------------------------------------------------- diff --git a/eval/src/apps/tensor_conformance/test_spec.json b/eval/src/apps/tensor_conformance/test_spec.json index 95439b0f104..24edc9a7ac7 100644 --- a/eval/src/apps/tensor_conformance/test_spec.json +++ b/eval/src/apps/tensor_conformance/test_spec.json @@ -532,6 +532,24 @@ {"expression":"map(a,f(a)(sigmoid(a)))","inputs":{"a":"0x010301780179017A180161036261720169BFF8000000000000016103626172016ABFF6666666666666016103626172016BBFF4CCCCCCCCCCCD016103626172016CBFF3333333333333016103666F6F0169BFFE666666666666016103666F6F016ABFFCCCCCCCCCCCCD016103666F6F016BBFFB333333333333016103666F6F016CBFF999999999999A0162036261720169BFE6666666666666016203626172016ABFE3333333333334016203626172016BBFE0000000000000016203626172016CBFD9999999999998016203666F6F0169BFF199999999999A016203666F6F016ABFF0000000000000016203666F6F016BBFECCCCCCCCCCCCC016203666F6F016CBFE999999999999A01630362617201693FB99999999999A0016303626172016A3FC99999999999A0016303626172016B3FD3333333333330016303626172016C3FD9999999999998016303666F6F0169BFD3333333333334016303666F6F016ABFC9999999999998016303666F6F016BBFB99999999999A0016303666F6F016C0000000000000000"},"result":{"expect":"0x010301780179017A1801610362617201693FC759B8355A1BB0016103626172016A3FC95209D0A1A2DE016103626172016B3FCB69C25FE3C688016103626172016C3FCDA0FADA5A6609016103666F6F01693FC0A764FD2927E7016103666F6F016A3FC2282CFA533F46016103666F6F016B3FC3C5848EF36C9E016103666F6F016C3FC5806BEB16EB7F01620362617201693FD53C695ABCD715016203626172016A3FD6AD912C137583016203626172016B3FD829A0565978DE016203626172016C3FD9AF19F3D3169C016203666F6F01693FCFF77A137CDBF9016203666F6F016A3FD136561454BA86016203666F6F016B3FD27FCDA8478FA3016203666F6F016C3FD3D775461EDE9501630362617201693FE0CCA12729AFB8016303626172016A3FE1983D7795F414016303626172016B3FE261D545E46A8A016303626172016C3FE32873061674B2016303666F6F01693FDB3C5574372AEB016303666F6F016A3FDCCF8510D417DA016303666F6F016B3FDE66BDB1ACA090016303666F6F016C3FE0000000000000"}} {"expression":"map(a,f(a)(sigmoid(a)))","inputs":{"a":"0x0301017902017803017A070203626172BFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE33333333333343FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A03666F6FBFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFE0000000000000BFD9999999999998BFD3333333333334BFC9999999999998BFB99999999999A000000000000000003FB99999999999A03FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF8000000000000"},"result":{"expect":"0x0301017902017803017A0702036261723FCDA0FADA5A66093FCFF77A137CDBF93FD136561454BA863FD27FCDA8478FA33FD3D775461EDE953FD53C695ABCD7153FD6AD912C1375833FE1983D7795F4143FE261D545E46A8A3FE32873061674B23FE3EB2FD4D343913FE4A93769F6453E3FE561CB52A194763FE614455CF090B63FEA9FE5053A45203FEB0E9EDC4324D83FEB75F4C16B302F3FEBD626C0B5B6063FEC2F7D5A8A79C93FEC8247621BC0C83FECCED80FEF120203666F6F3FC0A764FD2927E73FC2282CFA533F463FC3C5848EF36C9E3FC5806BEB16EB7F3FC759B8355A1BB03FC95209D0A1A2DE3FCB69C25FE3C6883FD829A0565978DE3FD9AF19F3D3169C3FDB3C5574372AEB3FDCCF8510D417DA3FDE66BDB1ACA0903FE00000000000003FE0CCA12729AFB83FE6C0192BDC382E3FE764D4F5D5A2BD3FE802217B20C9023FE897C14969667F3FE9258F68070E5E3FE9AB7D8BD797483FEA2991F2A97914"}} {"expression":"map(a,f(a)(sigmoid(a)))","inputs":{"a":"0x03020178017A010179050C01610169BFFE666666666666BFF8000000000000BFF199999999999ABFE6666666666666BFD33333333333340161016ABFFCCCCCCCCCCCCDBFF6666666666666BFF0000000000000BFE3333333333334BFC99999999999980161016BBFFB333333333333BFF4CCCCCCCCCCCDBFECCCCCCCCCCCCCBFE0000000000000BFB99999999999A00161016CBFF999999999999ABFF3333333333333BFE999999999999ABFD99999999999980000000000000000016201693FB99999999999A03FE00000000000003FECCCCCCCCCCCCC3FF4CCCCCCCCCCCC3FFB3333333333340162016A3FC99999999999A03FE33333333333343FF00000000000003FF66666666666663FFCCCCCCCCCCCCC0162016B3FD33333333333303FE66666666666683FF199999999999A3FF80000000000003FFE6666666666660162016C3FD99999999999983FE99999999999983FF33333333333343FF999999999999A4000000000000000016301694000CCCCCCCCCCCC40040000000000004007333333333334400A666666666666400D99999999999A0163016A400199999999999A4004CCCCCCCCCCCC4008000000000000400B333333333334400E6666666666660163016B4002666666666666400599999999999A4008CCCCCCCCCCCC400C000000000000400F3333333333340163016C40033333333333344006666666666666400999999999999A400CCCCCCCCCCCCC4010000000000000"},"result":{"expect":"0x03020178017A010179050C016101693FC0A764FD2927E73FC759B8355A1BB03FCFF77A137CDBF93FD53C695ABCD7153FDB3C5574372AEB0161016A3FC2282CFA533F463FC95209D0A1A2DE3FD136561454BA863FD6AD912C1375833FDCCF8510D417DA0161016B3FC3C5848EF36C9E3FCB69C25FE3C6883FD27FCDA8478FA33FD829A0565978DE3FDE66BDB1ACA0900161016C3FC5806BEB16EB7F3FCDA0FADA5A66093FD3D775461EDE953FD9AF19F3D3169C3FE0000000000000016201693FE0CCA12729AFB83FE3EB2FD4D343913FE6C0192BDC382E3FE9258F68070E5E3FEB0E9EDC4324D80162016A3FE1983D7795F4143FE4A93769F6453E3FE764D4F5D5A2BD3FE9AB7D8BD797483FEB75F4C16B302F0162016B3FE261D545E46A8A3FE561CB52A194763FE802217B20C9023FEA2991F2A979143FEBD626C0B5B6060162016C3FE32873061674B23FE614455CF090B63FE897C14969667F3FEA9FE5053A45203FEC2F7D5A8A79C9016301693FEC8247621BC0C83FED9291DDB596F83FEE54C20D06AA953FEEDC99CF2C9D4D3FEF3A59F801F5820163016A3FECCED80FEF12023FEDC99E39374D9C3FEE7B7CBC36FABD3FEEF76F8069F3FB3FEF4CBFA61DE6A30163016B3FED15854CD0D92B3FEDFC1F4CE6E8223FEE9EDD88B9D8AF3FEF0FDFCBF19A933FEF5D77DCF758080163016C3FED56A636946E583FEE2A667D67D08C3FEEBF2786AED6983FEF261E0FCD4B463FEF6CA82F0DE1EA"}} +{"expression":"elu(a)","inputs":{"a":"0x0200BFFE666666666666"},"result":{"expect":"0x0200BFEB36BBDEFDC9FA"}} +{"expression":"elu(a)","inputs":{"a":"0x0201017803BFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333"},"result":{"expect":"0x0201017803BFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7E"}} +{"expression":"elu(a)","inputs":{"a":"0x0202017803017905BFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE3333333333334BFE0000000000000"},"result":{"expect":"0x0202017803017905BFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7EBFE98A1050412C7BBFE8DC1E236D28F9BFE81BE0AF127E3BBFE7476B67B98EC0BFE65C9DF4C2F690BFE5591EBDB77208BFE43A54E4E98864BFE2FD619FFBC8F0BFE19F18DD3F123CBFE01BF92311555FBFDCE04528D3F63ABFD92E9A0720D3EC"}} +{"expression":"elu(a)","inputs":{"a":"0x0203017803017905017A07BFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE3333333333334BFE0000000000000BFD9999999999998BFD3333333333334BFC9999999999998BFB99999999999A000000000000000003FB99999999999A03FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF80000000000003FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A4002666666666666400333333333333440040000000000004004CCCCCCCCCCCC400599999999999A4006666666666666400733333333333440080000000000004008CCCCCCCCCCCC400999999999999A400A666666666666400B333333333334400C000000000000400CCCCCCCCCCCCC400D99999999999A400E666666666666400F333333333334401000000000000040106666666666664010CCCCCCCCCCCD4011333333333333401199999999999A401200000000000040126666666666664012CCCCCCCCCCCD4013333333333333401399999999999A401400000000000040146666666666664014CCCCCCCCCCCD4015333333333333401599999999999A401600000000000040166666666666664016CCCCCCCCCCCD4017333333333333401799999999999A401800000000000040186666666666664018CCCCCCCCCCCC4019333333333334401999999999999A401A000000000000401A666666666666401ACCCCCCCCCCCC401B333333333334401B99999999999A401C000000000000401C666666666666401CCCCCCCCCCCCC401D333333333334401D99999999999A401E000000000000401E666666666666401ECCCCCCCCCCCC401F333333333334401F99999999999A402000000000000040203333333333334020666666666666402099999999999A4020CCCCCCCCCCCD4021000000000000"},"result":{"expect":"0x0203017803017905017A07BFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7EBFE98A1050412C7BBFE8DC1E236D28F9BFE81BE0AF127E3BBFE7476B67B98EC0BFE65C9DF4C2F690BFE5591EBDB77208BFE43A54E4E98864BFE2FD619FFBC8F0BFE19F18DD3F123CBFE01BF92311555FBFDCE04528D3F63ABFD92E9A0720D3ECBFD51979F31B1E24BFD0966F2C7907F6BFC733D4A7A67A98BFB85C933156A63000000000000000003FB99999999999A03FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF80000000000003FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A4002666666666666400333333333333440040000000000004004CCCCCCCCCCCC400599999999999A4006666666666666400733333333333440080000000000004008CCCCCCCCCCCC400999999999999A400A666666666666400B333333333334400C000000000000400CCCCCCCCCCCCC400D99999999999A400E666666666666400F333333333334401000000000000040106666666666664010CCCCCCCCCCCD4011333333333333401199999999999A401200000000000040126666666666664012CCCCCCCCCCCD4013333333333333401399999999999A401400000000000040146666666666664014CCCCCCCCCCCD4015333333333333401599999999999A401600000000000040166666666666664016CCCCCCCCCCCD4017333333333333401799999999999A401800000000000040186666666666664018CCCCCCCCCCCC4019333333333334401999999999999A401A000000000000401A666666666666401ACCCCCCCCCCCC401B333333333334401B99999999999A401C000000000000401C666666666666401CCCCCCCCCCCCC401D333333333334401D99999999999A401E000000000000401E666666666666401ECCCCCCCCCCCC401F333333333334401F99999999999A402000000000000040203333333333334020666666666666402099999999999A4020CCCCCCCCCCCD4021000000000000"}} +{"expression":"elu(a)","inputs":{"a":"0x01010178030161BFFE6666666666660162BFFCCCCCCCCCCCCD0163BFFB333333333333"},"result":{"expect":"0x01010178030161BFEB36BBDEFDC9FA0162BFEAB5DF1B20BD730163BFEA2774E1D59D7E"}} +{"expression":"elu(a)","inputs":{"a":"0x01020178017906016103626172BFFCCCCCCCCCCCCD016103666F6FBFFE666666666666016203626172BFF999999999999A016203666F6FBFFB333333333333016303626172BFF6666666666666016303666F6FBFF8000000000000"},"result":{"expect":"0x01020178017906016103626172BFEAB5DF1B20BD73016103666F6FBFEB36BBDEFDC9FA016203626172BFE98A1050412C7B016203666F6FBFEA2774E1D59D7E016303626172BFE81BE0AF127E3B016303666F6FBFE8DC1E236D28F9"}} +{"expression":"elu(a)","inputs":{"a":"0x010301780179017A180161036261720169BFF8000000000000016103626172016ABFF6666666666666016103626172016BBFF4CCCCCCCCCCCD016103626172016CBFF3333333333333016103666F6F0169BFFE666666666666016103666F6F016ABFFCCCCCCCCCCCCD016103666F6F016BBFFB333333333333016103666F6F016CBFF999999999999A0162036261720169BFE6666666666666016203626172016ABFE3333333333334016203626172016BBFE0000000000000016203626172016CBFD9999999999998016203666F6F0169BFF199999999999A016203666F6F016ABFF0000000000000016203666F6F016BBFECCCCCCCCCCCCC016203666F6F016CBFE999999999999A01630362617201693FB99999999999A0016303626172016A3FC99999999999A0016303626172016B3FD3333333333330016303626172016C3FD9999999999998016303666F6F0169BFD3333333333334016303666F6F016ABFC9999999999998016303666F6F016BBFB99999999999A0016303666F6F016C0000000000000000"},"result":{"expect":"0x010301780179017A180161036261720169BFE8DC1E236D28F9016103626172016ABFE81BE0AF127E3B016103626172016BBFE7476B67B98EC0016103626172016CBFE65C9DF4C2F690016103666F6F0169BFEB36BBDEFDC9FA016103666F6F016ABFEAB5DF1B20BD73016103666F6F016BBFEA2774E1D59D7E016103666F6F016CBFE98A1050412C7B0162036261720169BFE01BF92311555F016203626172016ABFDCE04528D3F63A016203626172016BBFD92E9A0720D3EC016203626172016CBFD51979F31B1E24016203666F6F0169BFE5591EBDB77208016203666F6F016ABFE43A54E4E98864016203666F6F016BBFE2FD619FFBC8F0016203666F6F016CBFE19F18DD3F123C01630362617201693FB99999999999A0016303626172016A3FC99999999999A0016303626172016B3FD3333333333330016303626172016C3FD9999999999998016303666F6F0169BFD0966F2C7907F6016303666F6F016ABFC733D4A7A67A98016303666F6F016BBFB85C933156A630016303666F6F016C0000000000000000"}} +{"expression":"elu(a)","inputs":{"a":"0x0301017902017803017A070203626172BFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE33333333333343FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A03666F6FBFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFE0000000000000BFD9999999999998BFD3333333333334BFC9999999999998BFB99999999999A000000000000000003FB99999999999A03FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF8000000000000"},"result":{"expect":"0x0301017902017803017A070203626172BFE65C9DF4C2F690BFE5591EBDB77208BFE43A54E4E98864BFE2FD619FFBC8F0BFE19F18DD3F123CBFE01BF92311555FBFDCE04528D3F63A3FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A03666F6FBFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7EBFE98A1050412C7BBFE8DC1E236D28F9BFE81BE0AF127E3BBFE7476B67B98EC0BFD92E9A0720D3ECBFD51979F31B1E24BFD0966F2C7907F6BFC733D4A7A67A98BFB85C933156A63000000000000000003FB99999999999A03FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF8000000000000"}} +{"expression":"elu(a)","inputs":{"a":"0x03020178017A010179050C01610169BFFE666666666666BFF8000000000000BFF199999999999ABFE6666666666666BFD33333333333340161016ABFFCCCCCCCCCCCCDBFF6666666666666BFF0000000000000BFE3333333333334BFC99999999999980161016BBFFB333333333333BFF4CCCCCCCCCCCDBFECCCCCCCCCCCCCBFE0000000000000BFB99999999999A00161016CBFF999999999999ABFF3333333333333BFE999999999999ABFD99999999999980000000000000000016201693FB99999999999A03FE00000000000003FECCCCCCCCCCCCC3FF4CCCCCCCCCCCC3FFB3333333333340162016A3FC99999999999A03FE33333333333343FF00000000000003FF66666666666663FFCCCCCCCCCCCCC0162016B3FD33333333333303FE66666666666683FF199999999999A3FF80000000000003FFE6666666666660162016C3FD99999999999983FE99999999999983FF33333333333343FF999999999999A4000000000000000016301694000CCCCCCCCCCCC40040000000000004007333333333334400A666666666666400D99999999999A0163016A400199999999999A4004CCCCCCCCCCCC4008000000000000400B333333333334400E6666666666660163016B4002666666666666400599999999999A4008CCCCCCCCCCCC400C000000000000400F3333333333340163016C40033333333333344006666666666666400999999999999A400CCCCCCCCCCCCC4010000000000000"},"result":{"expect":"0x03020178017A010179050C01610169BFEB36BBDEFDC9FABFE8DC1E236D28F9BFE5591EBDB77208BFE01BF92311555FBFD0966F2C7907F60161016ABFEAB5DF1B20BD73BFE81BE0AF127E3BBFE43A54E4E98864BFDCE04528D3F63ABFC733D4A7A67A980161016BBFEA2774E1D59D7EBFE7476B67B98EC0BFE2FD619FFBC8F0BFD92E9A0720D3ECBFB85C933156A6300161016CBFE98A1050412C7BBFE65C9DF4C2F690BFE19F18DD3F123CBFD51979F31B1E240000000000000000016201693FB99999999999A03FE00000000000003FECCCCCCCCCCCCC3FF4CCCCCCCCCCCC3FFB3333333333340162016A3FC99999999999A03FE33333333333343FF00000000000003FF66666666666663FFCCCCCCCCCCCCC0162016B3FD33333333333303FE66666666666683FF199999999999A3FF80000000000003FFE6666666666660162016C3FD99999999999983FE99999999999983FF33333333333343FF999999999999A4000000000000000016301694000CCCCCCCCCCCC40040000000000004007333333333334400A666666666666400D99999999999A0163016A400199999999999A4004CCCCCCCCCCCC4008000000000000400B333333333334400E6666666666660163016B4002666666666666400599999999999A4008CCCCCCCCCCCC400C000000000000400F3333333333340163016C40033333333333344006666666666666400999999999999A400CCCCCCCCCCCCC4010000000000000"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x0200BFFE666666666666"},"result":{"expect":"0x0200BFEB36BBDEFDC9FA"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x0201017803BFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333"},"result":{"expect":"0x0201017803BFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7E"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x0202017803017905BFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE3333333333334BFE0000000000000"},"result":{"expect":"0x0202017803017905BFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7EBFE98A1050412C7BBFE8DC1E236D28F9BFE81BE0AF127E3BBFE7476B67B98EC0BFE65C9DF4C2F690BFE5591EBDB77208BFE43A54E4E98864BFE2FD619FFBC8F0BFE19F18DD3F123CBFE01BF92311555FBFDCE04528D3F63ABFD92E9A0720D3EC"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x0203017803017905017A07BFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE3333333333334BFE0000000000000BFD9999999999998BFD3333333333334BFC9999999999998BFB99999999999A000000000000000003FB99999999999A03FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF80000000000003FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A4002666666666666400333333333333440040000000000004004CCCCCCCCCCCC400599999999999A4006666666666666400733333333333440080000000000004008CCCCCCCCCCCC400999999999999A400A666666666666400B333333333334400C000000000000400CCCCCCCCCCCCC400D99999999999A400E666666666666400F333333333334401000000000000040106666666666664010CCCCCCCCCCCD4011333333333333401199999999999A401200000000000040126666666666664012CCCCCCCCCCCD4013333333333333401399999999999A401400000000000040146666666666664014CCCCCCCCCCCD4015333333333333401599999999999A401600000000000040166666666666664016CCCCCCCCCCCD4017333333333333401799999999999A401800000000000040186666666666664018CCCCCCCCCCCC4019333333333334401999999999999A401A000000000000401A666666666666401ACCCCCCCCCCCC401B333333333334401B99999999999A401C000000000000401C666666666666401CCCCCCCCCCCCC401D333333333334401D99999999999A401E000000000000401E666666666666401ECCCCCCCCCCCC401F333333333334401F99999999999A402000000000000040203333333333334020666666666666402099999999999A4020CCCCCCCCCCCD4021000000000000"},"result":{"expect":"0x0203017803017905017A07BFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7EBFE98A1050412C7BBFE8DC1E236D28F9BFE81BE0AF127E3BBFE7476B67B98EC0BFE65C9DF4C2F690BFE5591EBDB77208BFE43A54E4E98864BFE2FD619FFBC8F0BFE19F18DD3F123CBFE01BF92311555FBFDCE04528D3F63ABFD92E9A0720D3ECBFD51979F31B1E24BFD0966F2C7907F6BFC733D4A7A67A98BFB85C933156A63000000000000000003FB99999999999A03FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF80000000000003FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A4002666666666666400333333333333440040000000000004004CCCCCCCCCCCC400599999999999A4006666666666666400733333333333440080000000000004008CCCCCCCCCCCC400999999999999A400A666666666666400B333333333334400C000000000000400CCCCCCCCCCCCC400D99999999999A400E666666666666400F333333333334401000000000000040106666666666664010CCCCCCCCCCCD4011333333333333401199999999999A401200000000000040126666666666664012CCCCCCCCCCCD4013333333333333401399999999999A401400000000000040146666666666664014CCCCCCCCCCCD4015333333333333401599999999999A401600000000000040166666666666664016CCCCCCCCCCCD4017333333333333401799999999999A401800000000000040186666666666664018CCCCCCCCCCCC4019333333333334401999999999999A401A000000000000401A666666666666401ACCCCCCCCCCCC401B333333333334401B99999999999A401C000000000000401C666666666666401CCCCCCCCCCCCC401D333333333334401D99999999999A401E000000000000401E666666666666401ECCCCCCCCCCCC401F333333333334401F99999999999A402000000000000040203333333333334020666666666666402099999999999A4020CCCCCCCCCCCD4021000000000000"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x01010178030161BFFE6666666666660162BFFCCCCCCCCCCCCD0163BFFB333333333333"},"result":{"expect":"0x01010178030161BFEB36BBDEFDC9FA0162BFEAB5DF1B20BD730163BFEA2774E1D59D7E"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x01020178017906016103626172BFFCCCCCCCCCCCCD016103666F6FBFFE666666666666016203626172BFF999999999999A016203666F6FBFFB333333333333016303626172BFF6666666666666016303666F6FBFF8000000000000"},"result":{"expect":"0x01020178017906016103626172BFEAB5DF1B20BD73016103666F6FBFEB36BBDEFDC9FA016203626172BFE98A1050412C7B016203666F6FBFEA2774E1D59D7E016303626172BFE81BE0AF127E3B016303666F6FBFE8DC1E236D28F9"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x010301780179017A180161036261720169BFF8000000000000016103626172016ABFF6666666666666016103626172016BBFF4CCCCCCCCCCCD016103626172016CBFF3333333333333016103666F6F0169BFFE666666666666016103666F6F016ABFFCCCCCCCCCCCCD016103666F6F016BBFFB333333333333016103666F6F016CBFF999999999999A0162036261720169BFE6666666666666016203626172016ABFE3333333333334016203626172016BBFE0000000000000016203626172016CBFD9999999999998016203666F6F0169BFF199999999999A016203666F6F016ABFF0000000000000016203666F6F016BBFECCCCCCCCCCCCC016203666F6F016CBFE999999999999A01630362617201693FB99999999999A0016303626172016A3FC99999999999A0016303626172016B3FD3333333333330016303626172016C3FD9999999999998016303666F6F0169BFD3333333333334016303666F6F016ABFC9999999999998016303666F6F016BBFB99999999999A0016303666F6F016C0000000000000000"},"result":{"expect":"0x010301780179017A180161036261720169BFE8DC1E236D28F9016103626172016ABFE81BE0AF127E3B016103626172016BBFE7476B67B98EC0016103626172016CBFE65C9DF4C2F690016103666F6F0169BFEB36BBDEFDC9FA016103666F6F016ABFEAB5DF1B20BD73016103666F6F016BBFEA2774E1D59D7E016103666F6F016CBFE98A1050412C7B0162036261720169BFE01BF92311555F016203626172016ABFDCE04528D3F63A016203626172016BBFD92E9A0720D3EC016203626172016CBFD51979F31B1E24016203666F6F0169BFE5591EBDB77208016203666F6F016ABFE43A54E4E98864016203666F6F016BBFE2FD619FFBC8F0016203666F6F016CBFE19F18DD3F123C01630362617201693FB99999999999A0016303626172016A3FC99999999999A0016303626172016B3FD3333333333330016303626172016C3FD9999999999998016303666F6F0169BFD0966F2C7907F6016303666F6F016ABFC733D4A7A67A98016303666F6F016BBFB85C933156A630016303666F6F016C0000000000000000"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x0301017902017803017A070203626172BFF3333333333333BFF199999999999ABFF0000000000000BFECCCCCCCCCCCCCBFE999999999999ABFE6666666666666BFE33333333333343FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A03666F6FBFFE666666666666BFFCCCCCCCCCCCCDBFFB333333333333BFF999999999999ABFF8000000000000BFF6666666666666BFF4CCCCCCCCCCCDBFE0000000000000BFD9999999999998BFD3333333333334BFC9999999999998BFB99999999999A000000000000000003FB99999999999A03FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF8000000000000"},"result":{"expect":"0x0301017902017803017A070203626172BFE65C9DF4C2F690BFE5591EBDB77208BFE43A54E4E98864BFE2FD619FFBC8F0BFE19F18DD3F123CBFE01BF92311555FBFDCE04528D3F63A3FC99999999999A03FD33333333333303FD99999999999983FE00000000000003FE33333333333343FE66666666666683FE99999999999983FF999999999999A3FFB3333333333343FFCCCCCCCCCCCCC3FFE66666666666640000000000000004000CCCCCCCCCCCC400199999999999A03666F6FBFEB36BBDEFDC9FABFEAB5DF1B20BD73BFEA2774E1D59D7EBFE98A1050412C7BBFE8DC1E236D28F9BFE81BE0AF127E3BBFE7476B67B98EC0BFD92E9A0720D3ECBFD51979F31B1E24BFD0966F2C7907F6BFC733D4A7A67A98BFB85C933156A63000000000000000003FB99999999999A03FECCCCCCCCCCCCC3FF00000000000003FF199999999999A3FF33333333333343FF4CCCCCCCCCCCC3FF66666666666663FF8000000000000"}} +{"expression":"map(a,f(a)(elu(a)))","inputs":{"a":"0x03020178017A010179050C01610169BFFE666666666666BFF8000000000000BFF199999999999ABFE6666666666666BFD33333333333340161016ABFFCCCCCCCCCCCCDBFF6666666666666BFF0000000000000BFE3333333333334BFC99999999999980161016BBFFB333333333333BFF4CCCCCCCCCCCDBFECCCCCCCCCCCCCBFE0000000000000BFB99999999999A00161016CBFF999999999999ABFF3333333333333BFE999999999999ABFD99999999999980000000000000000016201693FB99999999999A03FE00000000000003FECCCCCCCCCCCCC3FF4CCCCCCCCCCCC3FFB3333333333340162016A3FC99999999999A03FE33333333333343FF00000000000003FF66666666666663FFCCCCCCCCCCCCC0162016B3FD33333333333303FE66666666666683FF199999999999A3FF80000000000003FFE6666666666660162016C3FD99999999999983FE99999999999983FF33333333333343FF999999999999A4000000000000000016301694000CCCCCCCCCCCC40040000000000004007333333333334400A666666666666400D99999999999A0163016A400199999999999A4004CCCCCCCCCCCC4008000000000000400B333333333334400E6666666666660163016B4002666666666666400599999999999A4008CCCCCCCCCCCC400C000000000000400F3333333333340163016C40033333333333344006666666666666400999999999999A400CCCCCCCCCCCCC4010000000000000"},"result":{"expect":"0x03020178017A010179050C01610169BFEB36BBDEFDC9FABFE8DC1E236D28F9BFE5591EBDB77208BFE01BF92311555FBFD0966F2C7907F60161016ABFEAB5DF1B20BD73BFE81BE0AF127E3BBFE43A54E4E98864BFDCE04528D3F63ABFC733D4A7A67A980161016BBFEA2774E1D59D7EBFE7476B67B98EC0BFE2FD619FFBC8F0BFD92E9A0720D3ECBFB85C933156A6300161016CBFE98A1050412C7BBFE65C9DF4C2F690BFE19F18DD3F123CBFD51979F31B1E240000000000000000016201693FB99999999999A03FE00000000000003FECCCCCCCCCCCCC3FF4CCCCCCCCCCCC3FFB3333333333340162016A3FC99999999999A03FE33333333333343FF00000000000003FF66666666666663FFCCCCCCCCCCCCC0162016B3FD33333333333303FE66666666666683FF199999999999A3FF80000000000003FFE6666666666660162016C3FD99999999999983FE99999999999983FF33333333333343FF999999999999A4000000000000000016301694000CCCCCCCCCCCC40040000000000004007333333333334400A666666666666400D99999999999A0163016A400199999999999A4004CCCCCCCCCCCC4008000000000000400B333333333334400E6666666666660163016B4002666666666666400599999999999A4008CCCCCCCCCCCC400C000000000000400F3333333333340163016C40033333333333344006666666666666400999999999999A400CCCCCCCCCCCCC4010000000000000"}} {"expression":"a in [1,5,7,13,42]","inputs":{"a":"0x02003FF0000000000000"},"result":{"expect":"0x02003FF0000000000000"}} {"expression":"a in [1,5,7,13,42]","inputs":{"a":"0x02010178033FF000000000000040000000000000004008000000000000"},"result":{"expect":"0x02010178033FF000000000000000000000000000000000000000000000"}} {"expression":"a in [1,5,7,13,42]","inputs":{"a":"0x02020178030179053FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C00000000000040200000000000004022000000000000402400000000000040260000000000004028000000000000402A000000000000402C000000000000402E000000000000"},"result":{"expect":"0x02020178030179053FF00000000000000000000000000000000000000000000000000000000000003FF000000000000000000000000000003FF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003FF000000000000000000000000000000000000000000000"}} @@ -1224,4 +1242,4 @@ {"expression":"tensor(x[10])(x+1)","inputs":{},"result":{"expect":"0x020101780A3FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C000000000000402000000000000040220000000000004024000000000000"}} {"expression":"tensor(x[5],y[4])(x*4+(y+1))","inputs":{},"result":{"expect":"0x02020178050179043FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C00000000000040200000000000004022000000000000402400000000000040260000000000004028000000000000402A000000000000402C000000000000402E00000000000040300000000000004031000000000000403200000000000040330000000000004034000000000000"}} {"expression":"tensor(x[5],y[4])(x==y)","inputs":{},"result":{"expect":"0x02020178050179043FF000000000000000000000000000000000000000000000000000000000000000000000000000003FF000000000000000000000000000000000000000000000000000000000000000000000000000003FF000000000000000000000000000000000000000000000000000000000000000000000000000003FF00000000000000000000000000000000000000000000000000000000000000000000000000000"}} -{"num_tests":1226} +{"num_tests":1244} diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp index a443ccb3d01..76f776df552 100644 --- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/eval_spec.h> #include <vespa/eval/eval/basic_nodes.h> @@ -181,9 +182,9 @@ TEST("require that dot product works with tensor function") { InterpretedFunction interpreted(engine, function, types); EXPECT_EQUAL(1u, interpreted.program_size()); InterpretedFunction::Context ctx(interpreted); - TensorValue va(engine.create(a)); - TensorValue vb(engine.create(b)); - InterpretedFunction::SimpleObjectParams params({va,vb}); + Value::UP va = engine.from_spec(a); + Value::UP vb = engine.from_spec(b); + InterpretedFunction::SimpleObjectParams params({*va,*vb}); const Value &result = interpreted.eval(ctx, params); EXPECT_TRUE(result.is_double()); EXPECT_EQUAL(expect, result.as_double()); @@ -211,12 +212,12 @@ TEST("require that matrix multiplication works with tensor function") { InterpretedFunction interpreted(engine, function, types); EXPECT_EQUAL(1u, interpreted.program_size()); InterpretedFunction::Context ctx(interpreted); - TensorValue va(engine.create(a)); - TensorValue vb(engine.create(b)); - InterpretedFunction::SimpleObjectParams params({va,vb}); + Value::UP va = engine.from_spec(a); + Value::UP vb = engine.from_spec(b); + InterpretedFunction::SimpleObjectParams params({*va,*vb}); const Value &result = interpreted.eval(ctx, params); ASSERT_TRUE(result.is_tensor()); - EXPECT_EQUAL(expect, engine.to_spec(*result.as_tensor())); + EXPECT_EQUAL(expect, engine.to_spec(result)); } //----------------------------------------------------------------------------- diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp index 0b6ea9e4d35..97b34f5be3c 100644 --- a/eval/src/tests/eval/node_types/node_types_test.cpp +++ b/eval/src/tests/eval/node_types/node_types_test.cpp @@ -221,6 +221,7 @@ TEST("require that various operations resolve appropriate type") { TEST_DO(verify_op1("isNan(%s)")); // IsNan TEST_DO(verify_op1("relu(%s)")); // Relu TEST_DO(verify_op1("sigmoid(%s)")); // Sigmoid + TEST_DO(verify_op1("elu(%s)")); // Elu } TEST("require that map resolves correct type") { diff --git a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp index 150b86f27ce..c3b42124155 100644 --- a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp +++ b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp @@ -13,7 +13,7 @@ using Cells = SimpleTensor::Cells; using Address = SimpleTensor::Address; using Stash = vespalib::Stash; -TensorSpec to_spec(const Tensor &a) { return a.engine().to_spec(a); } +TensorSpec to_spec(const Value &a) { return SimpleTensorEngine::ref().to_spec(a); } const Tensor &unwrap(const Value &value) { ASSERT_TRUE(value.is_tensor()); @@ -35,7 +35,7 @@ TEST("require that simple tensors can be built using tensor spec") { .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); - auto tensor = SimpleTensorEngine::ref().create(spec); + Value::UP tensor = SimpleTensorEngine::ref().from_spec(spec); TensorSpec full_spec("tensor(w{},x[2],y{},z[2])"); full_spec .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) @@ -54,7 +54,7 @@ TEST("require that simple tensors can be built using tensor spec") { .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); - auto full_tensor = SimpleTensorEngine::ref().create(full_spec); + Value::UP full_tensor = SimpleTensorEngine::ref().from_spec(full_spec); EXPECT_EQUAL(full_spec, to_spec(*tensor)); EXPECT_EQUAL(full_spec, to_spec(*full_tensor)); }; @@ -73,7 +73,7 @@ TEST("require that simple tensors can have their values negated") { auto result = tensor->map([](double a){ return -a; }); EXPECT_EQUAL(to_spec(*expect), to_spec(*result)); Stash stash; - const Value &result2 = SimpleTensorEngine::ref().map(TensorValue(*tensor), operation::Neg::f, stash); + const Value &result2 = SimpleTensorEngine::ref().map(*tensor, operation::Neg::f, stash); EXPECT_EQUAL(to_spec(*expect), to_spec(unwrap(result2))); } @@ -98,7 +98,7 @@ TEST("require that simple tensors can be multiplied with each other") { auto result = SimpleTensor::join(*lhs, *rhs, [](double a, double b){ return (a * b); }); EXPECT_EQUAL(to_spec(*expect), to_spec(*result)); Stash stash; - const Value &result2 = SimpleTensorEngine::ref().join(TensorValue(*lhs), TensorValue(*rhs), operation::Mul::f, stash); + const Value &result2 = SimpleTensorEngine::ref().join(*lhs, *rhs, operation::Mul::f, stash); EXPECT_EQUAL(to_spec(*expect), to_spec(unwrap(result2))); } @@ -129,10 +129,10 @@ TEST("require that simple tensors support dimension reduction") { EXPECT_EQUAL(to_spec(*expect_sum_y), to_spec(*result_sum_y)); EXPECT_EQUAL(to_spec(*expect_sum_x), to_spec(*result_sum_x)); EXPECT_EQUAL(to_spec(*expect_sum_all), to_spec(*result_sum_all)); - const Value &result_sum_y_2 = SimpleTensorEngine::ref().reduce(TensorValue(*tensor), Aggr::SUM, {"y"}, stash); - const Value &result_sum_x_2 = SimpleTensorEngine::ref().reduce(TensorValue(*tensor), Aggr::SUM, {"x"}, stash); - const Value &result_sum_all_2 = SimpleTensorEngine::ref().reduce(TensorValue(*tensor), Aggr::SUM, {"x", "y"}, stash); - const Value &result_sum_all_3 = SimpleTensorEngine::ref().reduce(TensorValue(*tensor), Aggr::SUM, {}, stash); + const Value &result_sum_y_2 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {"y"}, stash); + const Value &result_sum_x_2 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {"x"}, stash); + const Value &result_sum_all_2 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {"x", "y"}, stash); + const Value &result_sum_all_3 = SimpleTensorEngine::ref().reduce(*tensor, Aggr::SUM, {}, stash); EXPECT_EQUAL(to_spec(*expect_sum_y), to_spec(unwrap(result_sum_y_2))); EXPECT_EQUAL(to_spec(*expect_sum_x), to_spec(unwrap(result_sum_x_2))); EXPECT_TRUE(result_sum_all_2.is_double()); diff --git a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp index 8bd86621bf6..681a4dabc19 100644 --- a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp +++ b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp @@ -12,38 +12,37 @@ using namespace vespalib; using namespace vespalib::eval; using namespace vespalib::eval::tensor_function; -struct EvalCtx : TensorFunction::Input { +struct EvalCtx { const TensorEngine &engine; Stash stash; ErrorValue error; - std::map<size_t, Value::UP> tensors; + std::vector<Value::UP> tensors; + std::vector<Value::CREF> params; EvalCtx(const TensorEngine &engine_in) : engine(engine_in), stash(), error(), tensors() {} - ~EvalCtx() { } - void add_tensor(std::unique_ptr<Tensor> tensor, size_t id) { - tensors.emplace(id, std::make_unique<TensorValue>(std::move(tensor))); + ~EvalCtx() {} + size_t add_tensor(Value::UP tensor) { + size_t id = params.size(); + params.emplace_back(*tensor); + tensors.push_back(std::move(tensor)); + return id; } - const Value &get_tensor(size_t id) const override { - if (tensors.count(id) == 0) { - return error; - } - return *tensors.find(id)->second; + const Value &eval(const TensorFunction &fun) { + return fun.eval(params, stash); } - const Value &eval(const TensorFunction &fun) { return fun.eval(*this, stash); } - const ValueType type(const Tensor &tensor) const { return engine.type_of(tensor); } - TensorFunction::UP compile(tensor_function::Node_UP expr) const { - return engine.compile(std::move(expr)); + const TensorFunction &compile(const tensor_function::Node &expr) { + return engine.compile(expr, stash); } - std::unique_ptr<Tensor> make_tensor_inject() { - return engine.create( + Value::UP make_tensor_inject() { + return engine.from_spec( TensorSpec("tensor(x[2],y[2])") .add({{"x", 0}, {"y", 0}}, 1.0) .add({{"x", 0}, {"y", 1}}, 2.0) .add({{"x", 1}, {"y", 0}}, 3.0) .add({{"x", 1}, {"y", 1}}, 4.0)); } - std::unique_ptr<Tensor> make_tensor_reduce_input() { - return engine.create( + Value::UP make_tensor_reduce_input() { + return engine.from_spec( TensorSpec("tensor(x[3],y[2])") .add({{"x",0},{"y",0}}, 1) .add({{"x",1},{"y",0}}, 2) @@ -52,43 +51,43 @@ struct EvalCtx : TensorFunction::Input { .add({{"x",1},{"y",1}}, 5) .add({{"x",2},{"y",1}}, 6)); } - std::unique_ptr<Tensor> make_tensor_reduce_y_output() { - return engine.create( + Value::UP make_tensor_reduce_y_output() { + return engine.from_spec( TensorSpec("tensor(x[3])") .add({{"x",0}}, 5) .add({{"x",1}}, 7) .add({{"x",2}}, 9)); } - std::unique_ptr<Tensor> make_tensor_map_input() { - return engine.create( + Value::UP make_tensor_map_input() { + return engine.from_spec( TensorSpec("tensor(x{},y{})") .add({{"x","1"},{"y","1"}}, 1) .add({{"x","2"},{"y","1"}}, -3) .add({{"x","1"},{"y","2"}}, 5)); } - std::unique_ptr<Tensor> make_tensor_map_output() { - return engine.create( + Value::UP make_tensor_map_output() { + return engine.from_spec( TensorSpec("tensor(x{},y{})") .add({{"x","1"},{"y","1"}}, -1) .add({{"x","2"},{"y","1"}}, 3) .add({{"x","1"},{"y","2"}}, -5)); } - std::unique_ptr<Tensor> make_tensor_apply_lhs() { - return engine.create( + Value::UP make_tensor_apply_lhs() { + return engine.from_spec( TensorSpec("tensor(x{},y{})") .add({{"x","1"},{"y","1"}}, 1) .add({{"x","2"},{"y","1"}}, 3) .add({{"x","1"},{"y","2"}}, 5)); } - std::unique_ptr<Tensor> make_tensor_apply_rhs() { - return engine.create( + Value::UP make_tensor_apply_rhs() { + return engine.from_spec( TensorSpec("tensor(y{},z{})") .add({{"y","1"},{"z","1"}}, 7) .add({{"y","2"},{"z","1"}}, 11) .add({{"y","1"},{"z","2"}}, 13)); } - std::unique_ptr<Tensor> make_tensor_apply_output() { - return engine.create( + Value::UP make_tensor_apply_output() { + return engine.from_spec( TensorSpec("tensor(x{},y{},z{})") .add({{"x","1"},{"y","1"},{"z","1"}}, 7) .add({{"x","1"},{"y","1"},{"z","2"}}, 13) @@ -98,65 +97,69 @@ struct EvalCtx : TensorFunction::Input { } }; -void verify_equal(const Tensor &expect, const Value &value) { +void verify_equal(const Value &expect, const Value &value) { const Tensor *tensor = value.as_tensor(); ASSERT_TRUE(tensor != nullptr); - ASSERT_EQUAL(&expect.engine(), &tensor->engine()); - auto expect_spec = expect.engine().to_spec(expect); - auto value_spec = tensor->engine().to_spec(*tensor); + const Tensor *expect_tensor = expect.as_tensor(); + ASSERT_TRUE(expect_tensor != nullptr); + ASSERT_EQUAL(&expect_tensor->engine(), &tensor->engine()); + auto expect_spec = expect_tensor->engine().to_spec(expect); + auto value_spec = tensor->engine().to_spec(value); EXPECT_EQUAL(expect_spec, value_spec); } TEST("require that tensor injection works") { EvalCtx ctx(SimpleTensorEngine::ref()); - ctx.add_tensor(ctx.make_tensor_inject(), 1); - auto expect = ctx.make_tensor_inject(); - auto fun = inject(ValueType::from_spec("tensor(x[2],y[2])"), 1); - EXPECT_EQUAL(ctx.type(*expect), fun->result_type); - auto prog = ctx.compile(std::move(fun)); - TEST_DO(verify_equal(*expect, ctx.eval(*prog))); + size_t a_id = ctx.add_tensor(ctx.make_tensor_inject()); + Value::UP expect = ctx.make_tensor_inject(); + const auto &fun = inject(ValueType::from_spec("tensor(x[2],y[2])"), a_id, ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); } TEST("require that partial tensor reduction works") { EvalCtx ctx(SimpleTensorEngine::ref()); - ctx.add_tensor(ctx.make_tensor_reduce_input(), 1); - auto expect = ctx.make_tensor_reduce_y_output(); - auto fun = reduce(inject(ValueType::from_spec("tensor(x[3],y[2])"), 1), Aggr::SUM, {"y"}); - EXPECT_EQUAL(ctx.type(*expect), fun->result_type); - auto prog = ctx.compile(std::move(fun)); - TEST_DO(verify_equal(*expect, ctx.eval(*prog))); + size_t a_id = ctx.add_tensor(ctx.make_tensor_reduce_input()); + Value::UP expect = ctx.make_tensor_reduce_y_output(); + const auto &fun = reduce(inject(ValueType::from_spec("tensor(x[3],y[2])"), a_id, ctx.stash), Aggr::SUM, {"y"}, ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); } TEST("require that full tensor reduction works") { EvalCtx ctx(SimpleTensorEngine::ref()); - ctx.add_tensor(ctx.make_tensor_reduce_input(), 1); - auto fun = reduce(inject(ValueType::from_spec("tensor(x[3],y[2])"), 1), Aggr::SUM, {}); - EXPECT_EQUAL(ValueType::from_spec("double"), fun->result_type); - auto prog = ctx.compile(std::move(fun)); - EXPECT_EQUAL(21.0, ctx.eval(*prog).as_double()); + size_t a_id = ctx.add_tensor(ctx.make_tensor_reduce_input()); + const auto &fun = reduce(inject(ValueType::from_spec("tensor(x[3],y[2])"), a_id, ctx.stash), Aggr::SUM, {}, ctx.stash); + EXPECT_EQUAL(ValueType::from_spec("double"), fun.result_type); + const auto &prog = ctx.compile(fun); + const Value &result = ctx.eval(prog); + EXPECT_TRUE(result.is_double()); + EXPECT_EQUAL(21.0, result.as_double()); } TEST("require that tensor map works") { EvalCtx ctx(SimpleTensorEngine::ref()); - ctx.add_tensor(ctx.make_tensor_map_input(), 1); - auto expect = ctx.make_tensor_map_output(); - auto fun = map(inject(ValueType::from_spec("tensor(x{},y{})"), 1), operation::Neg::f); - EXPECT_EQUAL(ctx.type(*expect), fun->result_type); - auto prog = ctx.compile(std::move(fun)); - TEST_DO(verify_equal(*expect, ctx.eval(*prog))); + size_t a_id = ctx.add_tensor(ctx.make_tensor_map_input()); + Value::UP expect = ctx.make_tensor_map_output(); + const auto &fun = map(inject(ValueType::from_spec("tensor(x{},y{})"), a_id, ctx.stash), operation::Neg::f, ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); } TEST("require that tensor join works") { EvalCtx ctx(SimpleTensorEngine::ref()); - ctx.add_tensor(ctx.make_tensor_apply_lhs(), 1); - ctx.add_tensor(ctx.make_tensor_apply_rhs(), 2); - auto expect = ctx.make_tensor_apply_output(); - auto fun = join(inject(ValueType::from_spec("tensor(x{},y{})"), 1), - inject(ValueType::from_spec("tensor(y{},z{})"), 2), - operation::Mul::f); - EXPECT_EQUAL(ctx.type(*expect), fun->result_type); - auto prog = ctx.compile(std::move(fun)); - TEST_DO(verify_equal(*expect, ctx.eval(*prog))); + size_t a_id = ctx.add_tensor(ctx.make_tensor_apply_lhs()); + size_t b_id = ctx.add_tensor(ctx.make_tensor_apply_rhs()); + Value::UP expect = ctx.make_tensor_apply_output(); + const auto &fun = join(inject(ValueType::from_spec("tensor(x{},y{})"), a_id, ctx.stash), + inject(ValueType::from_spec("tensor(y{},z{})"), b_id, ctx.stash), + operation::Mul::f, ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp index ee8e502815f..5dd8caa6e27 100644 --- a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp +++ b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp @@ -3,59 +3,56 @@ #include <vespa/eval/eval/value_cache/constant_tensor_loader.h> #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/tensor.h> using namespace vespalib::eval; -std::unique_ptr<Tensor> dense_tensor_nocells() { - return SimpleTensorEngine::ref() - .create(TensorSpec("tensor(x[2],y[2])")); +TensorSpec sparse_tensor_nocells() { + return TensorSpec("tensor(x{},y{})"); } -std::unique_ptr<Tensor> make_nodim_tensor() { - return SimpleTensorEngine::ref() - .create(TensorSpec("double")); +TensorSpec make_dense_tensor() { + return TensorSpec("tensor(x[2],y[2])") + .add({{"x", 0}, {"y", 0}}, 1.0) + .add({{"x", 0}, {"y", 1}}, 2.0) + .add({{"x", 1}, {"y", 0}}, 3.0) + .add({{"x", 1}, {"y", 1}}, 4.0); } -std::unique_ptr<Tensor> make_dense_tensor() { - return SimpleTensorEngine::ref() - .create(TensorSpec("tensor(x[2],y[2])") - .add({{"x", 0}, {"y", 0}}, 1.0) - .add({{"x", 0}, {"y", 1}}, 2.0) - .add({{"x", 1}, {"y", 0}}, 3.0) - .add({{"x", 1}, {"y", 1}}, 4.0)); +TensorSpec make_sparse_tensor() { + return TensorSpec("tensor(x{},y{})") + .add({{"x", "foo"}, {"y", "bar"}}, 1.0) + .add({{"x", "bar"}, {"y", "foo"}}, 2.0); } -std::unique_ptr<Tensor> make_sparse_tensor() { - return SimpleTensorEngine::ref() - .create(TensorSpec("tensor(x{},y{})") - .add({{"x", "foo"}, {"y", "bar"}}, 1.0) - .add({{"x", "bar"}, {"y", "foo"}}, 2.0)); +TensorSpec make_mixed_tensor() { + return TensorSpec("tensor(x{},y[2])") + .add({{"x", "foo"}, {"y", 0}}, 1.0) + .add({{"x", "foo"}, {"y", 1}}, 2.0); } -std::unique_ptr<Tensor> make_mixed_tensor() { - return SimpleTensorEngine::ref() - .create(TensorSpec("tensor(x{},y[2])") - .add({{"x", "foo"}, {"y", 0}}, 1.0) - .add({{"x", "foo"}, {"y", 1}}, 2.0)); +void verify_tensor(const TensorSpec &expect, ConstantValue::UP actual) { + const auto &engine = SimpleTensorEngine::ref(); + ASSERT_EQUAL(expect.type(), actual->type().to_spec()); + ASSERT_TRUE(&engine == &actual->value().as_tensor()->engine()); + EXPECT_EQUAL(expect, engine.to_spec(actual->value())); } -void verify_tensor(std::unique_ptr<Tensor> expect, ConstantValue::UP actual) { - const auto &engine = expect->engine(); - ASSERT_EQUAL(engine.type_of(*expect), actual->type()); - ASSERT_TRUE(&engine == &actual->value().as_tensor()->engine()); - EXPECT_EQUAL(engine.to_spec(*expect), engine.to_spec(*actual->value().as_tensor())); +void verify_invalid(ConstantValue::UP actual) { + EXPECT_EQUAL(actual->type(), ValueType::double_type()); + EXPECT_EQUAL(actual->value().as_double(), 0.0); } TEST_F("require that invalid types loads an empty double", ConstantTensorLoader(SimpleTensorEngine::ref())) { - TEST_DO(verify_tensor(make_nodim_tensor(), f1.create(TEST_PATH("dense.json"), "invalid type spec"))); + TEST_DO(verify_invalid(f1.create(TEST_PATH("dense.json"), "invalid type spec"))); } TEST_F("require that invalid file name loads an empty tensor", ConstantTensorLoader(SimpleTensorEngine::ref())) { - TEST_DO(verify_tensor(dense_tensor_nocells(), f1.create(TEST_PATH("missing_file.json"), "tensor(x[2],y[2])"))); + TEST_DO(verify_tensor(sparse_tensor_nocells(), f1.create(TEST_PATH("missing_file.json"), "tensor(x{},y{})"))); } TEST_F("require that invalid json loads an empty tensor", ConstantTensorLoader(SimpleTensorEngine::ref())) { - TEST_DO(verify_tensor(dense_tensor_nocells(), f1.create(TEST_PATH("invalid.json"), "tensor(x[2],y[2])"))); + TEST_DO(verify_tensor(sparse_tensor_nocells(), f1.create(TEST_PATH("invalid.json"), "tensor(x{},y{})"))); } TEST_F("require that dense tensors can be loaded", ConstantTensorLoader(SimpleTensorEngine::ref())) { @@ -75,7 +72,7 @@ TEST_F("require that lz4 compressed sparse tensor can be loaded", ConstantTensor } TEST_F("require that bad lz4 file fails to load creating empty result", ConstantTensorLoader(SimpleTensorEngine::ref())) { - TEST_DO(verify_tensor(dense_tensor_nocells(), f1.create(TEST_PATH("bad_lz4.json.lz4"), "tensor(x[2],y[2])"))); + TEST_DO(verify_tensor(sparse_tensor_nocells(), f1.create(TEST_PATH("bad_lz4.json.lz4"), "tensor(x{},y{})"))); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index 17df3d21d0c..ca77997bac7 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -50,34 +50,26 @@ asDenseTensor(const tensor::Tensor &tensor) return dynamic_cast<const DenseTensor &>(tensor); } -class FunctionInput : public TensorFunction::Input +class FunctionInput { private: tensor::Tensor::UP _lhsTensor; tensor::Tensor::UP _rhsTensor; const DenseTensor &_lhsDenseTensor; const DenseTensor &_rhsDenseTensor; - TensorValue _lhsValue; - TensorValue _rhsValue; + std::vector<Value::CREF> _params; public: FunctionInput(size_t lhsNumCells, size_t rhsNumCells) : _lhsTensor(makeTensor(lhsNumCells, 3.0)), _rhsTensor(makeTensor(rhsNumCells, 5.0)), _lhsDenseTensor(asDenseTensor(*_lhsTensor)), - _rhsDenseTensor(asDenseTensor(*_rhsTensor)), - _lhsValue(std::make_unique<DenseTensor>(_lhsDenseTensor.type(), - _lhsDenseTensor.cells())), - _rhsValue(std::make_unique<DenseTensor>(_rhsDenseTensor.type(), - _rhsDenseTensor.cells())) - {} - virtual const Value &get_tensor(size_t id) const override { - if (id == 0) { - return _lhsValue; - } else { - return _rhsValue; - } + _rhsDenseTensor(asDenseTensor(*_rhsTensor)) + { + _params.emplace_back(_lhsDenseTensor); + _params.emplace_back(_rhsDenseTensor); } + ConstArrayRef<Value::CREF> get() const { return _params; } double expectedDotProduct() const { return calcDotProduct(_lhsDenseTensor, _rhsDenseTensor); } @@ -91,11 +83,11 @@ struct Fixture ~Fixture(); double eval() const { Stash stash; - const Value &result = function.eval(input, stash); + const Value &result = function.eval(input.get(), stash); ASSERT_TRUE(result.is_double()); LOG(info, "eval(): (%s) * (%s) = %f", - input.get_tensor(0).type().to_spec().c_str(), - input.get_tensor(1).type().to_spec().c_str(), + input.get()[0].get().type().to_spec().c_str(), + input.get()[1].get().type().to_spec().c_str(), result.as_double()); return result.as_double(); } diff --git a/eval/src/tests/tensor/dense_tensor_function_compiler/dense_tensor_function_compiler_test.cpp b/eval/src/tests/tensor/dense_tensor_function_compiler/dense_tensor_function_compiler_test.cpp index 6dcfc0791e7..63829650cc5 100644 --- a/eval/src/tests/tensor/dense_tensor_function_compiler/dense_tensor_function_compiler_test.cpp +++ b/eval/src/tests/tensor/dense_tensor_function_compiler/dense_tensor_function_compiler_test.cpp @@ -3,32 +3,36 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/tensor/dense/dense_dot_product_function.h> #include <vespa/eval/tensor/dense/dense_tensor_function_compiler.h> +#include <vespa/eval/eval/operation.h> using namespace vespalib::eval; using namespace vespalib::eval::operation; using namespace vespalib::eval::tensor_function; using namespace vespalib::tensor; +using vespalib::Stash; template <typename T> const T *as(const TensorFunction &function) { return dynamic_cast<const T *>(&function); } -TensorFunction::UP +const TensorFunction & compileDotProduct(const vespalib::string &lhsType, - const vespalib::string &rhsType) + const vespalib::string &rhsType, + Stash &stash) { - Node_UP reduceNode = reduce(join(inject(ValueType::from_spec(lhsType), 1), - inject(ValueType::from_spec(rhsType), 3), - Mul::f), - Aggr::SUM, {}); - return DenseTensorFunctionCompiler::compile(std::move(reduceNode)); + const Node &reduceNode = reduce(join(inject(ValueType::from_spec(lhsType), 1, stash), + inject(ValueType::from_spec(rhsType), 3, stash), + Mul::f, stash), + Aggr::SUM, {}, stash); + return DenseTensorFunctionCompiler::compile(reduceNode, stash); } void assertCompiledDotProduct(const vespalib::string &lhsType, const vespalib::string &rhsType) { - TensorFunction::UP func = compileDotProduct(lhsType, rhsType); - const DenseDotProductFunction *dotProduct = as<DenseDotProductFunction>(*func); + Stash stash; + const TensorFunction &func = compileDotProduct(lhsType, rhsType, stash); + const DenseDotProductFunction *dotProduct = as<DenseDotProductFunction>(func); ASSERT_TRUE(dotProduct); EXPECT_EQUAL(1u, dotProduct->lhsTensorId()); EXPECT_EQUAL(3u, dotProduct->rhsTensorId()); @@ -38,8 +42,9 @@ void assertNotCompiledDotProduct(const vespalib::string &lhsType, const vespalib::string &rhsType) { - TensorFunction::UP func = compileDotProduct(lhsType, rhsType); - const Reduce *reduce = as<Reduce>(*func); + Stash stash; + const TensorFunction &func = compileDotProduct(lhsType, rhsType, stash); + const Reduce *reduce = as<Reduce>(func); EXPECT_TRUE(reduce); } diff --git a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp index c26429f47e4..e369e09b99a 100644 --- a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp @@ -9,6 +9,7 @@ #include <vespa/eval/eval/simple_tensor.h> using vespalib::eval::ValueType; +using vespalib::eval::Value; using vespalib::eval::TensorSpec; using vespalib::eval::SimpleTensor; using namespace vespalib::tensor; @@ -21,8 +22,8 @@ void verify_wrapped(const TensorSpec &source, const vespalib::string &type, cons } void verify(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) { - auto tensor = DefaultTensorEngine::ref().create(source); - const Tensor *tensor_impl = dynamic_cast<const Tensor *>(tensor.get()); + Value::UP value = DefaultTensorEngine::ref().from_spec(source); + const Tensor *tensor_impl = dynamic_cast<const Tensor *>(value->as_tensor()); ASSERT_TRUE(tensor_impl); TensorMapper mapper(ValueType::from_spec(type)); auto mapped = mapper.map(*tensor_impl); diff --git a/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp b/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp index 3daaa3f79b3..2ed0021b5c7 100644 --- a/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp +++ b/eval/src/tests/tensor/tensor_performance/tensor_performance_test.cpp @@ -25,21 +25,12 @@ const vespalib::string matrix_product_expr = "reduce(reduce((query+documen //----------------------------------------------------------------------------- -Value::UP wrap(std::unique_ptr<eval::Tensor> tensor) { - return Value::UP(new TensorValue(std::move(tensor))); -} - -//----------------------------------------------------------------------------- - struct Params { std::map<vespalib::string, Value::UP> map; Params &add(const vespalib::string &name, Value::UP value) { map.emplace(name, std::move(value)); return *this; } - Params &add(const vespalib::string &name, std::unique_ptr<eval::Tensor> value) { - return add(name, wrap(std::move(value))); - } }; InterpretedFunction::SimpleObjectParams make_params(const Function &function, const Params ¶ms) @@ -49,7 +40,7 @@ InterpretedFunction::SimpleObjectParams make_params(const Function &function, co for (size_t i = 0; i < function.num_params(); ++i) { auto param = params.map.find(function.param_name(i)); ASSERT_TRUE(param != params.map.end()); - fun_params.params.push_back(*(param->second)); + fun_params.params.push_back(*param->second); } return fun_params; } @@ -92,9 +83,8 @@ double benchmark_expression_us(const vespalib::string &expression, const Params //----------------------------------------------------------------------------- -tensor::Tensor::UP make_tensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().create(spec); - return tensor::Tensor::UP(dynamic_cast<tensor::Tensor*>(tensor.release())); +Value::UP make_tensor(TensorSpec spec) { + return DefaultTensorEngine::ref().from_spec(spec); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/call_nodes.cpp b/eval/src/vespa/eval/eval/call_nodes.cpp index 0e54ed183f4..69a9151a2bb 100644 --- a/eval/src/vespa/eval/eval/call_nodes.cpp +++ b/eval/src/vespa/eval/eval/call_nodes.cpp @@ -41,6 +41,7 @@ CallRepo::CallRepo() : _map() { add(nodes::IsNan()); add(nodes::Relu()); add(nodes::Sigmoid()); + add(nodes::Elu()); } } // namespace vespalib::eval::nodes diff --git a/eval/src/vespa/eval/eval/call_nodes.h b/eval/src/vespa/eval/eval/call_nodes.h index 4c5611a863a..8210616750e 100644 --- a/eval/src/vespa/eval/eval/call_nodes.h +++ b/eval/src/vespa/eval/eval/call_nodes.h @@ -137,6 +137,7 @@ struct Max : CallHelper<Max> { Max() : Helper("max", 2) {} }; struct IsNan : CallHelper<IsNan> { IsNan() : Helper("isNan", 1) {} }; struct Relu : CallHelper<Relu> { Relu() : Helper("relu", 1) {} }; struct Sigmoid : CallHelper<Sigmoid> { Sigmoid() : Helper("sigmoid", 1) {} }; +struct Elu : CallHelper<Elu> { Elu() : Helper("elu", 1) {} }; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index f99c4ace2dd..cfe989e95f8 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -5,6 +5,7 @@ #include "node_traverser.h" #include "check_type.h" #include "tensor_spec.h" +#include "operation.h" #include <vespa/vespalib/util/classname.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/vespalib/util/benchmark_timer.h> @@ -111,39 +112,26 @@ void op_tensor_concat(State &state, uint64_t param) { //----------------------------------------------------------------------------- template <typename T> -const T &undef_cref() { +const T &undef_cref() { const T *undef = nullptr; assert(undef); return *undef; } struct TensorFunctionArgArgMeta { - TensorFunction::UP function; + const TensorFunction &function; size_t param1; size_t param2; - TensorFunctionArgArgMeta(TensorFunction::UP function_in, size_t param1_in, size_t param2_in) - : function(std::move(function_in)), param1(param1_in), param2(param2_in) {} -}; - -struct ArgArgInput : TensorFunction::Input { - const TensorFunctionArgArgMeta &meta; - State &state; - ArgArgInput(const TensorFunctionArgArgMeta &meta_in, State &state_in) - : meta(meta_in), state(state_in) {} - const Value &get_tensor(size_t id) const override { - if (id == 0) { - return state.params->resolve(meta.param1, state.stash); - } else if (id == 1) { - return state.params->resolve(meta.param2, state.stash); - } - return undef_cref<Value>(); - } + TensorFunctionArgArgMeta(const TensorFunction &function_in, size_t param1_in, size_t param2_in) + : function(function_in), param1(param1_in), param2(param2_in) {} }; void op_tensor_function_arg_arg(State &state, uint64_t param) { const TensorFunctionArgArgMeta &meta = unwrap_param<TensorFunctionArgArgMeta>(param); - ArgArgInput input(meta, state); - state.stack.push_back(meta.function->eval(input, state.stash)); + Value::CREF params[2] = + {state.params->resolve(meta.param1, state.stash), + state.params->resolve(meta.param2, state.stash)}; + state.stack.push_back(meta.function.eval(ConstArrayRef<Value::CREF>(params, 2), state.stash)); } //----------------------------------------------------------------------------- @@ -279,12 +267,12 @@ struct ProgramBuilder : public NodeVisitor, public NodeTraverser { program.pop_back(); // load auto a = as<Symbol>(node.get_child(0).get_child(0)); auto b = as<Symbol>(node.get_child(0).get_child(1)); - auto ir = tensor_function::reduce(tensor_function::join( - tensor_function::inject(types.get_type(*a), 0), - tensor_function::inject(types.get_type(*b), 1), - operation::Mul::f), node.aggr(), node.dimensions()); - auto fun = tensor_engine.compile(std::move(ir)); - const auto &meta = stash.create<TensorFunctionArgArgMeta>(std::move(fun), a->id(), b->id()); + const auto &ir = tensor_function::reduce(tensor_function::join( + tensor_function::inject(types.get_type(*a), 0, stash), + tensor_function::inject(types.get_type(*b), 1, stash), + operation::Mul::f, stash), node.aggr(), node.dimensions(), stash); + const auto &fun = tensor_engine.compile(ir, stash); + const auto &meta = stash.create<TensorFunctionArgArgMeta>(fun, a->id(), b->id()); program.emplace_back(op_tensor_function_arg_arg, wrap_param<TensorFunctionArgArgMeta>(meta)); } else { ReduceParams ¶ms = stash.create<ReduceParams>(node.aggr(), node.dimensions()); @@ -309,8 +297,7 @@ struct ProgramBuilder : public NodeVisitor, public NodeTraverser { } spec.add(addr, fun(¶ms[0])); } while (step_labels(params, type)); - auto tensor = tensor_engine.create(spec); - make_const_op(node, stash.create<TensorValue>(std::move(tensor))); + make_const_op(node, *stash.create<Value::UP>(tensor_engine.from_spec(spec))); } void visit(const TensorConcat &node) override { vespalib::string &dimension = stash.create<vespalib::string>(node.dimension()); @@ -436,6 +423,9 @@ struct ProgramBuilder : public NodeVisitor, public NodeTraverser { void visit(const Sigmoid &node) override { make_map_op(node, operation::Sigmoid::f); } + void visit(const Elu &node) override { + make_map_op(node, operation::Elu::f); + } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/key_gen.cpp b/eval/src/vespa/eval/eval/key_gen.cpp index e0494e1fe11..86908b331ba 100644 --- a/eval/src/vespa/eval/eval/key_gen.cpp +++ b/eval/src/vespa/eval/eval/key_gen.cpp @@ -81,6 +81,7 @@ struct KeyGen : public NodeVisitor, public NodeTraverser { void visit(const IsNan &) override { add_byte(58); } void visit(const Relu &) override { add_byte(59); } void visit(const Sigmoid &) override { add_byte(60); } + void visit(const Elu &) override { add_byte(61); } // traverse bool open(const Node &node) override { node.accept(*this); return true; } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 9355cf7a4e4..f314f8a69cb 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -25,6 +25,7 @@ double vespalib_eval_isnan(double a) { return (std::isnan(a) ? 1.0 : 0.0); } double vespalib_eval_approx(double a, double b) { return (vespalib::approx_equal(a, b) ? 1.0 : 0.0); } double vespalib_eval_relu(double a) { return std::max(a, 0.0); } double vespalib_eval_sigmoid(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } +double vespalib_eval_elu(double a) { return (a < 0) ? std::exp(a) - 1.0 : a; } using vespalib::eval::gbdt::Forest; using resolve_function = double (*)(void *ctx, size_t idx); @@ -586,6 +587,9 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Sigmoid &) override { make_call_1("vespalib_eval_sigmoid"); } + void visit(const Elu &) override { + make_call_1("vespalib_eval_elu"); + } }; FunctionBuilder::~FunctionBuilder() { } @@ -628,7 +632,7 @@ LLVMWrapper::LLVMWrapper() size_t LLVMWrapper::make_function(size_t num_params, PassParams pass_params, const Node &root, const gbdt::Optimize::Chain &forest_optimizers) -{ +{ std::lock_guard<std::recursive_mutex> guard(_global_llvm_lock); size_t function_id = _functions.size(); FunctionBuilder builder(*_context, *_module, diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h index d3011a54ec0..6860be922f4 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h @@ -18,6 +18,7 @@ extern "C" { double vespalib_eval_approx(double a, double b); double vespalib_eval_relu(double a); double vespalib_eval_sigmoid(double a); + double vespalib_eval_elu(double a); }; namespace vespalib { diff --git a/eval/src/vespa/eval/eval/node_types.cpp b/eval/src/vespa/eval/eval/node_types.cpp index f86c3e1a84a..0cbc30667f0 100644 --- a/eval/src/vespa/eval/eval/node_types.cpp +++ b/eval/src/vespa/eval/eval/node_types.cpp @@ -164,6 +164,7 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser { void visit(const IsNan &node) override { resolve_op1(node); } void visit(const Relu &node) override { resolve_op1(node); } void visit(const Sigmoid &node) override { resolve_op1(node); } + void visit(const Elu &node) override { resolve_op1(node); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_visitor.h b/eval/src/vespa/eval/eval/node_visitor.h index c5a6fd51373..10b389db792 100644 --- a/eval/src/vespa/eval/eval/node_visitor.h +++ b/eval/src/vespa/eval/eval/node_visitor.h @@ -79,6 +79,7 @@ struct NodeVisitor { virtual void visit(const nodes::IsNan &) = 0; virtual void visit(const nodes::Relu &) = 0; virtual void visit(const nodes::Sigmoid &) = 0; + virtual void visit(const nodes::Elu &) = 0; virtual ~NodeVisitor() {} }; @@ -142,6 +143,7 @@ struct EmptyNodeVisitor : NodeVisitor { void visit(const nodes::IsNan &) override {} void visit(const nodes::Relu &) override {} void visit(const nodes::Sigmoid &) override {} + void visit(const nodes::Elu &) override {} }; } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index 42b1a110497..d697db40e7b 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -49,6 +49,7 @@ double Max::f(double a, double b) { return std::max(a, b); } double IsNan::f(double a) { return std::isnan(a) ? 1.0 : 0.0; } double Relu::f(double a) { return std::max(a, 0.0); } double Sigmoid::f(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } +double Elu::f(double a) { return (a < 0) ? std::exp(a) - 1 : a; } } // namespace vespalib::eval::operation } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/operation.h b/eval/src/vespa/eval/eval/operation.h index 12de7c3deb7..52a0fbabd22 100644 --- a/eval/src/vespa/eval/eval/operation.h +++ b/eval/src/vespa/eval/eval/operation.h @@ -51,6 +51,7 @@ struct Max { static double f(double a, double b); }; struct IsNan { static double f(double a); }; struct Relu { static double f(double a); }; struct Sigmoid { static double f(double a); }; +struct Elu { static double f(double a); }; } // namespace vespalib::eval::operation } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index e39e926708d..0e58d292334 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -604,7 +604,11 @@ SimpleTensor::rename(const std::vector<vespalib::string> &from, const std::vecto std::unique_ptr<SimpleTensor> SimpleTensor::create(const TensorSpec &spec) { - Builder builder(ValueType::from_spec(spec.type())); + ValueType my_type = ValueType::from_spec(spec.type()); + if (my_type.is_error()) { + return std::make_unique<SimpleTensor>(); + } + Builder builder(my_type); for (const auto &cell: spec.cells()) { builder.set(cell.first, cell.second); } diff --git a/eval/src/vespa/eval/eval/simple_tensor.h b/eval/src/vespa/eval/eval/simple_tensor.h index 366796f00d8..45d1853824d 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.h +++ b/eval/src/vespa/eval/eval/simple_tensor.h @@ -82,7 +82,7 @@ public: explicit SimpleTensor(double value); SimpleTensor(const ValueType &type_in, Cells cells_in); double as_double() const final override; - const ValueType &type() const { return _type; } + const ValueType &type() const override { return _type; } const Cells &cells() const { return _cells; } std::unique_ptr<SimpleTensor> map(map_fun_t function) const; std::unique_ptr<SimpleTensor> reduce(Aggregator &aggr, const std::vector<vespalib::string> &dimensions) const; diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp index 21498ca2ff1..2b3c5679488 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp @@ -10,7 +10,7 @@ namespace eval { namespace { -const SimpleTensor &to_simple(const Tensor &tensor) { +const SimpleTensor &as_simple(const Tensor &tensor) { assert(&tensor.engine() == &SimpleTensorEngine::ref()); return static_cast<const SimpleTensor&>(tensor); } @@ -20,98 +20,92 @@ const SimpleTensor &to_simple(const Value &value, Stash &stash) { return stash.create<SimpleTensor>(value.as_double()); } if (auto tensor = value.as_tensor()) { - return to_simple(*tensor); + return as_simple(*tensor); } return stash.create<SimpleTensor>(); // error } +template <typename F> +void with_simple(const Value &value, const F &f) { + if (value.is_double()) { + f(SimpleTensor(value.as_double())); + } else if (auto tensor = value.as_tensor()) { + f(as_simple(*tensor)); + } else { + f(SimpleTensor()); + } +} + const Value &to_value(std::unique_ptr<SimpleTensor> tensor, Stash &stash) { + if (tensor->type().is_tensor()) { + return *stash.create<Value::UP>(std::move(tensor)); + } if (tensor->type().is_double()) { - assert(tensor->cells().size() == 1u); - return stash.create<DoubleValue>(tensor->cells()[0].value); + return stash.create<DoubleValue>(tensor->as_double()); } + assert(tensor->type().is_error()); + return ErrorValue::instance; +} + +Value::UP to_value(std::unique_ptr<SimpleTensor> tensor) { if (tensor->type().is_tensor()) { - return stash.create<TensorValue>(std::move(tensor)); + return std::move(tensor); + } + if (tensor->type().is_double()) { + return std::make_unique<DoubleValue>(tensor->as_double()); } assert(tensor->type().is_error()); - return stash.create<ErrorValue>(); + return std::make_unique<ErrorValue>(); } } // namespace vespalib::eval::<unnamed> const SimpleTensorEngine SimpleTensorEngine::_engine; -ValueType -SimpleTensorEngine::type_of(const Tensor &tensor) const -{ - return to_simple(tensor).type(); -} - -vespalib::string -SimpleTensorEngine::to_string(const Tensor &tensor) const -{ - const SimpleTensor &simple_tensor = to_simple(tensor); - vespalib::string out = vespalib::make_string("simple(%s) {\n", simple_tensor.type().to_spec().c_str()); - for (const auto &cell: simple_tensor.cells()) { - size_t n = 0; - out.append(" ["); - for (const auto &label: cell.address) { - if (n++) { - out.append(","); - } - if (label.is_mapped()) { - out.append(label.name); - } else { - out.append(vespalib::make_string("%zu", label.index)); - } - } - out.append(vespalib::make_string("]: %g\n", cell.value)); - } - out.append("}"); - return out; -} +//----------------------------------------------------------------------------- TensorSpec -SimpleTensorEngine::to_spec(const Tensor &tensor) const +SimpleTensorEngine::to_spec(const Value &value) const { - const SimpleTensor &simple_tensor = to_simple(tensor); - ValueType type = simple_tensor.type(); - const auto &dimensions = type.dimensions(); - TensorSpec spec(type.to_spec()); - for (const auto &cell: simple_tensor.cells()) { - TensorSpec::Address addr; - assert(cell.address.size() == dimensions.size()); - for (size_t i = 0; i < cell.address.size(); ++i) { - const auto &label = cell.address[i]; - if (label.is_mapped()) { - addr.emplace(dimensions[i].name, TensorSpec::Label(label.name)); - } else { - addr.emplace(dimensions[i].name, TensorSpec::Label(label.index)); - } - } - spec.add(addr, cell.value); - } + TensorSpec spec(value.type().to_spec()); + const auto &dimensions = value.type().dimensions(); + with_simple(value, [&spec,&dimensions](const SimpleTensor &simple_tensor) + { + for (const auto &cell: simple_tensor.cells()) { + TensorSpec::Address addr; + assert(cell.address.size() == dimensions.size()); + for (size_t i = 0; i < cell.address.size(); ++i) { + const auto &label = cell.address[i]; + if (label.is_mapped()) { + addr.emplace(dimensions[i].name, TensorSpec::Label(label.name)); + } else { + addr.emplace(dimensions[i].name, TensorSpec::Label(label.index)); + } + } + spec.add(addr, cell.value); + } + }); return spec; } -std::unique_ptr<eval::Tensor> -SimpleTensorEngine::create(const TensorSpec &spec) const +Value::UP +SimpleTensorEngine::from_spec(const TensorSpec &spec) const { - return SimpleTensor::create(spec); + return to_value(SimpleTensor::create(spec)); } //----------------------------------------------------------------------------- void -SimpleTensorEngine::encode(const Value &value, nbostream &output, Stash &stash) const +SimpleTensorEngine::encode(const Value &value, nbostream &output) const { - SimpleTensor::encode(to_simple(value, stash), output); + with_simple(value, [&output](const SimpleTensor &tensor) { SimpleTensor::encode(tensor, output); }); } -const Value & -SimpleTensorEngine::decode(nbostream &input, Stash &stash) const +Value::UP +SimpleTensorEngine::decode(nbostream &input) const { - return to_value(SimpleTensor::decode(input), stash); + return to_value(SimpleTensor::decode(input)); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.h b/eval/src/vespa/eval/eval/simple_tensor_engine.h index c751f2f6b49..4cfd389dfa9 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.h +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.h @@ -19,14 +19,12 @@ private: public: static const TensorEngine &ref() { return _engine; }; - ValueType type_of(const Tensor &tensor) const override; - vespalib::string to_string(const Tensor &tensor) const override; - TensorSpec to_spec(const Tensor &tensor) const override; + TensorSpec to_spec(const Value &value) const override; + Value::UP from_spec(const TensorSpec &spec) const override; - std::unique_ptr<Tensor> create(const TensorSpec &spec) const override; + void encode(const Value &value, nbostream &output) const override; + Value::UP decode(nbostream &input) const override; - void encode(const Value &value, nbostream &output, Stash &stash) const override; - const Value &decode(nbostream &input, Stash &stash) const override; const Value &map(const Value &a, map_fun_t function, Stash &stash) const override; const Value &join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const override; const Value &reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const override; diff --git a/eval/src/vespa/eval/eval/tensor.cpp b/eval/src/vespa/eval/eval/tensor.cpp index 926606f8e26..645208ba8fb 100644 --- a/eval/src/vespa/eval/eval/tensor.cpp +++ b/eval/src/vespa/eval/eval/tensor.cpp @@ -18,7 +18,7 @@ operator==(const Tensor &lhs, const Tensor &rhs) std::ostream & operator<<(std::ostream &out, const Tensor &tensor) { - out << tensor.engine().to_string(tensor); + out << tensor.engine().to_spec(tensor).to_string(); return out; } diff --git a/eval/src/vespa/eval/eval/tensor.h b/eval/src/vespa/eval/eval/tensor.h index 57cd9abe1f5..149e2774bfb 100644 --- a/eval/src/vespa/eval/eval/tensor.h +++ b/eval/src/vespa/eval/eval/tensor.h @@ -3,6 +3,7 @@ #pragma once #include "value_type.h" +#include "value.h" namespace vespalib { namespace eval { @@ -18,7 +19,7 @@ class TensorEngine; * engine. TensorEngines should only have a single static instance per * implementation. **/ -class Tensor +class Tensor : public Value { private: const TensorEngine &_engine; @@ -30,7 +31,8 @@ public: Tensor(Tensor &&) = delete; Tensor &operator=(const Tensor &) = delete; Tensor &operator=(Tensor &&) = delete; - virtual double as_double() const = 0; + bool is_tensor() const override { return true; } + const Tensor *as_tensor() const override { return this; } const TensorEngine &engine() const { return _engine; } virtual ~Tensor() {} }; diff --git a/eval/src/vespa/eval/eval/tensor_engine.h b/eval/src/vespa/eval/eval/tensor_engine.h index 00927f0c1b1..02a7f0c655a 100644 --- a/eval/src/vespa/eval/eval/tensor_engine.h +++ b/eval/src/vespa/eval/eval/tensor_engine.h @@ -32,25 +32,23 @@ class TensorSpec; **/ struct TensorEngine { - using ValueType = eval::ValueType; + using Aggr = eval::Aggr; using Tensor = eval::Tensor; + using TensorFunction = eval::TensorFunction; using TensorSpec = eval::TensorSpec; using Value = eval::Value; - using map_fun_t = double (*)(double); + using ValueType = eval::ValueType; using join_fun_t = double (*)(double, double); - using Aggr = eval::Aggr; + using map_fun_t = double (*)(double); - virtual ValueType type_of(const Tensor &tensor) const = 0; - virtual vespalib::string to_string(const Tensor &tensor) const = 0; - virtual TensorSpec to_spec(const Tensor &tensor) const = 0; + virtual TensorSpec to_spec(const Value &value) const = 0; + virtual Value::UP from_spec(const TensorSpec &spec) const = 0; - virtual TensorFunction::UP compile(tensor_function::Node_UP expr) const { return std::move(expr); } + virtual void encode(const Value &value, nbostream &output) const = 0; + virtual Value::UP decode(nbostream &input) const = 0; - virtual std::unique_ptr<Tensor> create(const TensorSpec &spec) const = 0; + virtual const TensorFunction &compile(const tensor_function::Node &expr, Stash &) const { return expr; } - // havardpe: new API, WIP - virtual void encode(const Value &value, nbostream &output, Stash &stash) const = 0; - virtual const Value &decode(nbostream &input, Stash &stash) const = 0; virtual const Value &map(const Value &a, map_fun_t function, Stash &stash) const = 0; virtual const Value &join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const = 0; virtual const Value &reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const = 0; diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 0dcc930087f..9cd7c7fc9c2 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -5,68 +5,73 @@ #include "operation.h" #include "tensor.h" #include "tensor_engine.h" +#include "simple_tensor_engine.h" namespace vespalib { namespace eval { namespace tensor_function { -void Inject::accept(TensorFunctionVisitor &visitor) const { visitor.visit(*this); } -void Reduce::accept(TensorFunctionVisitor &visitor) const { visitor.visit(*this); } -void Map ::accept(TensorFunctionVisitor &visitor) const { visitor.visit(*this); } -void Join ::accept(TensorFunctionVisitor &visitor) const { visitor.visit(*this); } +const TensorEngine &infer_engine(const std::initializer_list<Value::CREF> &values) { + for (const Value &value: values) { + if (auto tensor = value.as_tensor()) { + return tensor->engine(); + } + } + return SimpleTensorEngine::ref(); +} //----------------------------------------------------------------------------- const Value & -Inject::eval(const Input &input, Stash &) const +Inject::eval(ConstArrayRef<Value::CREF> params, Stash &) const { - return input.get_tensor(tensor_id); + return params[tensor_id]; } const Value & -Reduce::eval(const Input &input, Stash &stash) const +Reduce::eval(ConstArrayRef<Value::CREF> params, Stash &stash) const { - const Value &a = tensor->eval(input, stash); - const TensorEngine &engine = a.as_tensor()->engine(); + const Value &a = tensor.eval(params, stash); + const TensorEngine &engine = infer_engine({a}); return engine.reduce(a, aggr, dimensions, stash); } const Value & -Map::eval(const Input &input, Stash &stash) const +Map::eval(ConstArrayRef<Value::CREF> params, Stash &stash) const { - const Value &a = tensor->eval(input, stash); - const TensorEngine &engine = a.as_tensor()->engine(); + const Value &a = tensor.eval(params, stash); + const TensorEngine &engine = infer_engine({a}); return engine.map(a, function, stash); } const Value & -Join::eval(const Input &input, Stash &stash) const +Join::eval(ConstArrayRef<Value::CREF> params, Stash &stash) const { - const Value &a = lhs_tensor->eval(input, stash); - const Value &b = rhs_tensor->eval(input, stash); - const TensorEngine &engine = a.as_tensor()->engine(); + const Value &a = lhs_tensor.eval(params, stash); + const Value &b = rhs_tensor.eval(params, stash); + const TensorEngine &engine = infer_engine({a,b}); return engine.join(a, b, function, stash); } //----------------------------------------------------------------------------- -Node_UP inject(const ValueType &type, size_t tensor_id) { - return std::make_unique<Inject>(type, tensor_id); +const Node &inject(const ValueType &type, size_t tensor_id, Stash &stash) { + return stash.create<Inject>(type, tensor_id); } -Node_UP reduce(Node_UP tensor, Aggr aggr, const std::vector<vespalib::string> &dimensions) { - ValueType result_type = tensor->result_type.reduce(dimensions); - return std::make_unique<Reduce>(result_type, std::move(tensor), aggr, dimensions); +const Node &reduce(const Node &tensor, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) { + ValueType result_type = tensor.result_type.reduce(dimensions); + return stash.create<Reduce>(result_type, tensor, aggr, dimensions); } -Node_UP map(Node_UP tensor, map_fun_t function) { - ValueType result_type = tensor->result_type; - return std::make_unique<Map>(result_type, std::move(tensor), function); +const Node &map(const Node &tensor, map_fun_t function, Stash &stash) { + ValueType result_type = tensor.result_type; + return stash.create<Map>(result_type, tensor, function); } -Node_UP join(Node_UP lhs_tensor, Node_UP rhs_tensor, join_fun_t function) { - ValueType result_type = ValueType::join(lhs_tensor->result_type, rhs_tensor->result_type); - return std::make_unique<Join>(result_type, std::move(lhs_tensor), std::move(rhs_tensor), function); +const Node &join(const Node &lhs_tensor, const Node &rhs_tensor, join_fun_t function, Stash &stash) { + ValueType result_type = ValueType::join(lhs_tensor.result_type, rhs_tensor.result_type); + return stash.create<Join>(result_type, lhs_tensor, rhs_tensor, function); } } // namespace vespalib::eval::tensor_function diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index cff21b7b9aa..359cabc18a0 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -5,8 +5,9 @@ #include <memory> #include <vector> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> #include "value_type.h" -#include "operation.h" +#include "value.h" #include "aggr.h" namespace vespalib { @@ -15,7 +16,6 @@ class Stash; namespace eval { -class Value; class Tensor; //----------------------------------------------------------------------------- @@ -29,28 +29,19 @@ class Tensor; **/ struct TensorFunction { - typedef std::unique_ptr<TensorFunction> UP; - /** - * Interface used to obtain input to a tensor function. - **/ - struct Input { - virtual const Value &get_tensor(size_t id) const = 0; - virtual ~Input() {} - }; - - /** - * Evaluate this tensor function based on the given input. The - * given stash can be used to store temporary objects that need to - * be kept alive for the return value to be valid. The return - * value must conform to the result type indicated by the - * intermediate representation describing this tensor function. + * Evaluate this tensor function based on the given + * parameters. The given stash can be used to store temporary + * objects that need to be kept alive for the return value to be + * valid. The return value must conform to the result type + * indicated by the intermediate representation describing this + * tensor function. * * @return result of evaluating this tensor function - * @param input external stuff needed to evaluate this function + * @param params external values needed to evaluate this function + * @param stash heterogeneous object store **/ - virtual const Value &eval(const Input &input, Stash &stash) const = 0; - + virtual const Value &eval(ConstArrayRef<Value::CREF> params, Stash &stash) const = 0; virtual ~TensorFunction() {} }; @@ -78,15 +69,13 @@ using join_fun_t = double (*)(double, double); **/ struct Node : public TensorFunction { - ValueType result_type; + const ValueType result_type; Node(const ValueType &result_type_in) : result_type(result_type_in) {} - virtual void accept(TensorFunctionVisitor &visitor) const = 0; Node(const Node &) = delete; Node &operator=(const Node &) = delete; Node(Node &&) = delete; Node &operator=(Node &&) = delete; }; -using Node_UP = std::unique_ptr<Node>; /** * Simple typecasting utility. @@ -95,68 +84,53 @@ template <typename T> const T *as(const Node &node) { return dynamic_cast<const T *>(&node); } struct Inject : Node { - size_t tensor_id; + const size_t tensor_id; Inject(const ValueType &result_type_in, size_t tensor_id_in) : Node(result_type_in), tensor_id(tensor_id_in) {} - void accept(TensorFunctionVisitor &visitor) const override; - const Value &eval(const Input &input, Stash &) const override; + const Value &eval(ConstArrayRef<Value::CREF> params, Stash &) const override; }; struct Reduce : Node { - Node_UP tensor; - Aggr aggr; - std::vector<vespalib::string> dimensions; + const Node &tensor; + const Aggr aggr; + const std::vector<vespalib::string> dimensions; Reduce(const ValueType &result_type_in, - Node_UP tensor_in, + const Node &tensor_in, Aggr aggr_in, const std::vector<vespalib::string> &dimensions_in) - : Node(result_type_in), tensor(std::move(tensor_in)), aggr(aggr_in), dimensions(dimensions_in) {} - void accept(TensorFunctionVisitor &visitor) const override; - const Value &eval(const Input &input, Stash &stash) const override; + : Node(result_type_in), tensor(tensor_in), aggr(aggr_in), dimensions(dimensions_in) {} + const Value &eval(ConstArrayRef<Value::CREF> params, Stash &stash) const override; }; struct Map : Node { - Node_UP tensor; - map_fun_t function; + const Node &tensor; + const map_fun_t function; Map(const ValueType &result_type_in, - Node_UP tensor_in, + const Node &tensor_in, map_fun_t function_in) - : Node(result_type_in), tensor(std::move(tensor_in)), function(function_in) {} - void accept(TensorFunctionVisitor &visitor) const override; - const Value &eval(const Input &input, Stash &stash) const override; + : Node(result_type_in), tensor(tensor_in), function(function_in) {} + const Value &eval(ConstArrayRef<Value::CREF> params, Stash &stash) const override; }; struct Join : Node { - Node_UP lhs_tensor; - Node_UP rhs_tensor; - join_fun_t function; + const Node &lhs_tensor; + const Node &rhs_tensor; + const join_fun_t function; Join(const ValueType &result_type_in, - Node_UP lhs_tensor_in, - Node_UP rhs_tensor_in, + const Node &lhs_tensor_in, + const Node &rhs_tensor_in, join_fun_t function_in) - : Node(result_type_in), lhs_tensor(std::move(lhs_tensor_in)), - rhs_tensor(std::move(rhs_tensor_in)), function(function_in) {} - void accept(TensorFunctionVisitor &visitor) const override; - const Value &eval(const Input &input, Stash &stash) const override; + : Node(result_type_in), lhs_tensor(lhs_tensor_in), + rhs_tensor(rhs_tensor_in), function(function_in) {} + const Value &eval(ConstArrayRef<Value::CREF> params, Stash &stash) const override; }; -Node_UP inject(const ValueType &type, size_t tensor_id); -Node_UP reduce(Node_UP tensor, Aggr aggr, const std::vector<vespalib::string> &dimensions); -Node_UP map(Node_UP tensor, map_fun_t function); -Node_UP join(Node_UP lhs_tensor, Node_UP rhs_tensor, join_fun_t function); +const Node &inject(const ValueType &type, size_t tensor_id, Stash &stash); +const Node &reduce(const Node &tensor, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash); +const Node &map(const Node &tensor, map_fun_t function, Stash &stash); +const Node &join(const Node &lhs_tensor, const Node &rhs_tensor, join_fun_t function, Stash &stash); } // namespace vespalib::eval::tensor_function - -struct TensorFunctionVisitor { - virtual void visit(const tensor_function::Inject &) = 0; - virtual void visit(const tensor_function::Reduce &) = 0; - virtual void visit(const tensor_function::Map &) = 0; - virtual void visit(const tensor_function::Join &) = 0; - virtual ~TensorFunctionVisitor() {} -}; - -//----------------------------------------------------------------------------- - } // namespace vespalib::eval } // namespace vespalib diff --git a/eval/src/vespa/eval/eval/test/eval_spec.cpp b/eval/src/vespa/eval/eval/test/eval_spec.cpp index d214486cf21..baa3ee989d4 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.cpp +++ b/eval/src/vespa/eval/eval/test/eval_spec.cpp @@ -150,6 +150,7 @@ EvalSpec::add_function_call_cases() { .add_case({my_nan}, 1.0).add_case({my_inf}, 0.0).add_case({-my_inf}, 0.0); add_rule({"a", -1.0, 1.0}, "relu(a)", [](double a){ return std::max(a, 0.0); }); add_rule({"a", -1.0, 1.0}, "sigmoid(a)", [](double a){ return 1.0 / (1.0 + std::exp(-1.0 * a)); }); + add_rule({"a", -1.0, 1.0}, "elu(a)", [](double a){ return (a < 0) ? std::exp(a)-1 : a; }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "atan2(a,b)", [](double a, double b){ return std::atan2(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "ldexp(a,b)", [](double a, double b){ return std::ldexp(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "pow(a,b)", [](double a, double b){ return std::pow(a, b); }); diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index e0a9f731804..23562f4a186 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -38,18 +38,17 @@ struct Eval { double _number; TensorSpec _tensor; public: - Result(const Value &value) : _type(Type::ERROR), _number(error_value), _tensor("error") { + Result() : _type(Type::ERROR), _number(error_value), _tensor("error") {} + Result(const TensorEngine &engine, const Value &value) : _type(Type::ERROR), _number(error_value), _tensor("error") { if (value.is_double()) { _type = Type::NUMBER; - _number = value.as_double(); - _tensor = TensorSpec("double").add({}, _number); - } else if (value.is_tensor()) { + } + if (value.is_tensor()) { + EXPECT_TRUE(_type == Type::ERROR); _type = Type::TENSOR; - _tensor = value.as_tensor()->engine().to_spec(*value.as_tensor()); - if (_tensor.type() == "double") { - _number = as_double(_tensor); - } } + _number = value.as_double(); + _tensor = engine.to_spec(value); } bool is_error() const { return (_type == Type::ERROR); } bool is_number() const { return (_type == Type::NUMBER); } @@ -60,20 +59,20 @@ struct Eval { } const TensorSpec &tensor() const { EXPECT_TRUE(is_tensor()); - return _tensor; + return _tensor; } }; virtual Result eval(const TensorEngine &) const { TEST_ERROR("wrong signature"); - return Result(ErrorValue()); + return Result(); } virtual Result eval(const TensorEngine &, const TensorSpec &) const { TEST_ERROR("wrong signature"); - return Result(ErrorValue()); + return Result(); } virtual Result eval(const TensorEngine &, const TensorSpec &, const TensorSpec &) const { TEST_ERROR("wrong signature"); - return Result(ErrorValue()); + return Result(); } virtual ~Eval() {} }; @@ -87,7 +86,7 @@ struct SafeEval : Eval { return unsafe.eval(engine); } catch (std::exception &e) { TEST_ERROR(e.what()); - return Result(ErrorValue()); + return Result(); } } Result eval(const TensorEngine &engine, const TensorSpec &a) const override { @@ -95,7 +94,7 @@ struct SafeEval : Eval { return unsafe.eval(engine, a); } catch (std::exception &e) { TEST_ERROR(e.what()); - return Result(ErrorValue()); + return Result(); } } @@ -104,7 +103,7 @@ struct SafeEval : Eval { return unsafe.eval(engine, a, b); } catch (std::exception &e) { TEST_ERROR(e.what()); - return Result(ErrorValue()); + return Result(); } } }; @@ -125,7 +124,7 @@ struct Expr_V : Eval { InterpretedFunction ifun(engine, fun, types); InterpretedFunction::Context ctx(ifun); InterpretedFunction::SimpleObjectParams params({}); - return Result(check_type(ifun.eval(ctx, params), types.get_type(fun.root()))); + return Result(engine, check_type(ifun.eval(ctx, params), types.get_type(fun.root()))); } }; @@ -139,9 +138,9 @@ struct Expr_T : Eval { NodeTypes types(fun, {a_type}); InterpretedFunction ifun(engine, fun, types); InterpretedFunction::Context ctx(ifun); - TensorValue va(engine.create(a)); - InterpretedFunction::SimpleObjectParams params({va}); - return Result(check_type(ifun.eval(ctx, params), types.get_type(fun.root()))); + Value::UP va = engine.from_spec(a); + InterpretedFunction::SimpleObjectParams params({*va}); + return Result(engine, check_type(ifun.eval(ctx, params), types.get_type(fun.root()))); } }; @@ -156,19 +155,15 @@ struct Expr_TT : Eval { NodeTypes types(fun, {a_type, b_type}); InterpretedFunction ifun(engine, fun, types); InterpretedFunction::Context ctx(ifun); - TensorValue va(engine.create(a)); - TensorValue vb(engine.create(b)); - InterpretedFunction::SimpleObjectParams params({va,vb}); - return Result(check_type(ifun.eval(ctx, params), types.get_type(fun.root()))); + Value::UP va = engine.from_spec(a); + Value::UP vb = engine.from_spec(b); + InterpretedFunction::SimpleObjectParams params({*va,*vb}); + return Result(engine, check_type(ifun.eval(ctx, params), types.get_type(fun.root()))); } }; const Value &make_value(const TensorEngine &engine, const TensorSpec &spec, Stash &stash) { - if (spec.type() == "double") { - double number = as_double(spec); - return stash.create<DoubleValue>(number); - } - return stash.create<TensorValue>(engine.create(spec)); + return *stash.create<Value::UP>(engine.from_spec(spec)); } //----------------------------------------------------------------------------- @@ -179,11 +174,11 @@ struct ImmediateReduce : Eval { std::vector<vespalib::string> dimensions; ImmediateReduce(Aggr aggr_in) : aggr(aggr_in), dimensions() {} ImmediateReduce(Aggr aggr_in, const vespalib::string &dimension) - : aggr(aggr_in), dimensions({dimension}) {} + : aggr(aggr_in), dimensions({dimension}) {} Result eval(const TensorEngine &engine, const TensorSpec &a) const override { Stash stash; const auto &lhs = make_value(engine, a, stash); - return Result(engine.reduce(lhs, aggr, dimensions, stash)); + return Result(engine, engine.reduce(lhs, aggr, dimensions, stash)); } }; @@ -195,7 +190,7 @@ struct ImmediateMap : Eval { Result eval(const TensorEngine &engine, const TensorSpec &a) const override { Stash stash; const auto &lhs = make_value(engine, a, stash); - return Result(engine.map(lhs, function, stash)); + return Result(engine, engine.map(lhs, function, stash)); } }; @@ -208,7 +203,7 @@ struct ImmediateJoin : Eval { Stash stash; const auto &lhs = make_value(engine, a, stash); const auto &rhs = make_value(engine, b, stash); - return Result(engine.join(lhs, rhs, function, stash)); + return Result(engine, engine.join(lhs, rhs, function, stash)); } }; @@ -220,7 +215,7 @@ struct ImmediateConcat : Eval { Stash stash; const auto &lhs = make_value(engine, a, stash); const auto &rhs = make_value(engine, b, stash); - return Result(engine.concat(lhs, rhs, dimension, stash)); + return Result(engine, engine.concat(lhs, rhs, dimension, stash)); } }; @@ -233,7 +228,7 @@ struct ImmediateRename : Eval { Result eval(const TensorEngine &engine, const TensorSpec &a) const override { Stash stash; const auto &lhs = make_value(engine, a, stash); - return Result(engine.rename(lhs, from, to, stash)); + return Result(engine, engine.rename(lhs, from, to, stash)); } }; @@ -243,20 +238,28 @@ const size_t tensor_id_a = 11; const size_t tensor_id_b = 12; // input used when evaluating in retained mode -struct Input : TensorFunction::Input { - std::vector<TensorValue> tensors; - Input(std::unique_ptr<Tensor> a) : tensors() { - tensors.emplace_back(std::move(a)); +struct Input { + std::vector<Value::UP> tensors; + std::vector<Value::CREF> params; + ~Input() {} + void pad_params() { + for (size_t i = 0; i < tensor_id_a; ++i) { + params.push_back(ErrorValue::instance); + } } - Input(std::unique_ptr<Tensor> a, std::unique_ptr<Tensor> b) : tensors() { - tensors.emplace_back(std::move(a)); - tensors.emplace_back(std::move(b)); + Input(Value::UP a) : tensors() { + pad_params(); + tensors.push_back(std::move(a)); + params.emplace_back(*tensors.back()); } - const Value &get_tensor(size_t id) const override { - size_t offset = (id - tensor_id_a); - ASSERT_GREATER(tensors.size(), offset); - return tensors[offset]; + Input(Value::UP a, Value::UP b) : tensors() { + pad_params(); + tensors.push_back(std::move(a)); + params.emplace_back(*tensors.back()); + tensors.push_back(std::move(b)); + params.emplace_back(*tensors.back()); } + ConstArrayRef<Value::CREF> get() const { return params; } }; // evaluate tensor reduce operation using tensor engine retained api @@ -267,13 +270,13 @@ struct RetainedReduce : Eval { RetainedReduce(Aggr aggr_in, const vespalib::string &dimension) : aggr(aggr_in), dimensions({dimension}) {} Result eval(const TensorEngine &engine, const TensorSpec &a) const override { - auto a_type = ValueType::from_spec(a.type()); - auto ir = tensor_function::reduce(tensor_function::inject(a_type, tensor_id_a), aggr, dimensions); - ValueType expect_type = ir->result_type; - auto fun = engine.compile(std::move(ir)); - Input input(engine.create(a)); Stash stash; - return Result(check_type(fun->eval(input, stash), expect_type)); + auto a_type = ValueType::from_spec(a.type()); + const auto &ir = tensor_function::reduce(tensor_function::inject(a_type, tensor_id_a, stash), aggr, dimensions, stash); + ValueType expect_type = ir.result_type; + const auto &fun = engine.compile(ir, stash); + Input input(engine.from_spec(a)); + return Result(engine, check_type(fun.eval(input.get(), stash), expect_type)); } }; @@ -282,13 +285,13 @@ struct RetainedMap : Eval { map_fun_t function; RetainedMap(map_fun_t function_in) : function(function_in) {} Result eval(const TensorEngine &engine, const TensorSpec &a) const override { - auto a_type = ValueType::from_spec(a.type()); - auto ir = tensor_function::map(tensor_function::inject(a_type, tensor_id_a), function); - ValueType expect_type = ir->result_type; - auto fun = engine.compile(std::move(ir)); - Input input(engine.create(a)); Stash stash; - return Result(check_type(fun->eval(input, stash), expect_type)); + auto a_type = ValueType::from_spec(a.type()); + const auto &ir = tensor_function::map(tensor_function::inject(a_type, tensor_id_a, stash), function, stash); + ValueType expect_type = ir.result_type; + const auto &fun = engine.compile(ir, stash); + Input input(engine.from_spec(a)); + return Result(engine, check_type(fun.eval(input.get(), stash), expect_type)); } }; @@ -297,16 +300,16 @@ struct RetainedJoin : Eval { join_fun_t function; RetainedJoin(join_fun_t function_in) : function(function_in) {} Result eval(const TensorEngine &engine, const TensorSpec &a, const TensorSpec &b) const override { + Stash stash; auto a_type = ValueType::from_spec(a.type()); auto b_type = ValueType::from_spec(b.type()); - auto ir = tensor_function::join(tensor_function::inject(a_type, tensor_id_a), - tensor_function::inject(b_type, tensor_id_b), - function); - ValueType expect_type = ir->result_type; - auto fun = engine.compile(std::move(ir)); - Input input(engine.create(a), engine.create(b)); - Stash stash; - return Result(check_type(fun->eval(input, stash), expect_type)); + const auto &ir = tensor_function::join(tensor_function::inject(a_type, tensor_id_a, stash), + tensor_function::inject(b_type, tensor_id_b, stash), + function, stash); + ValueType expect_type = ir.result_type; + const auto &fun = engine.compile(ir, stash); + Input input(engine.from_spec(a), engine.from_spec(b)); + return Result(engine, check_type(fun.eval(input.get(), stash), expect_type)); } }; @@ -369,21 +372,15 @@ struct TestContext { TestContext(const vespalib::string &module_path_in, const TensorEngine &engine_in) : module_path(module_path_in), ref_engine(SimpleTensorEngine::ref()), engine(engine_in) {} - std::unique_ptr<Tensor> tensor(const TensorSpec &spec) { - auto result = engine.create(spec); - EXPECT_EQUAL(spec.type(), engine.type_of(*result).to_spec()); - return result; - } - //------------------------------------------------------------------------- void verify_create_type(const vespalib::string &type_spec) { - auto tensor = engine.create(TensorSpec(type_spec)); - EXPECT_TRUE(&engine == &tensor->engine()); - EXPECT_EQUAL(type_spec, engine.type_of(*tensor).to_spec()); + Value::UP value = engine.from_spec(TensorSpec(type_spec)); + EXPECT_EQUAL(type_spec, value->type().to_spec()); } void test_tensor_create_type() { + TEST_DO(verify_create_type("error")); TEST_DO(verify_create_type("double")); TEST_DO(verify_create_type("tensor(x{})")); TEST_DO(verify_create_type("tensor(x{},y{})")); @@ -491,6 +488,7 @@ struct TestContext { TEST_DO(test_map_op("isNan(a)", operation::IsNan::f, Mask2Seq(SkipNth(3), 1.0, my_nan))); TEST_DO(test_map_op("relu(a)", operation::Relu::f, Sub2(Div10(N())))); TEST_DO(test_map_op("sigmoid(a)", operation::Sigmoid::f, Sub2(Div10(N())))); + TEST_DO(test_map_op("elu(a)", operation::Elu::f, Sub2(Div10(N())))); TEST_DO(test_map_op("a in [1,5,7,13,42]", MyIn::f, N())); TEST_DO(test_map_op("(a+1)*2", MyOp::f, Div10(N()))); } @@ -731,7 +729,7 @@ struct TestContext { TEST_STATE(make_string("lhs shape: %s, rhs shape: %s", lhs_input.type().c_str(), rhs_input.type().c_str()).c_str()); - Eval::Result expect = ImmediateJoin(op).eval(ref_engine, lhs_input, rhs_input); + Eval::Result expect = ImmediateJoin(op).eval(ref_engine, lhs_input, rhs_input); TEST_DO(verify_result(safe(eval).eval(engine, lhs_input, rhs_input), expect)); } TEST_DO(test_fixed_sparse_cases_apply_op(eval, op)); @@ -867,8 +865,8 @@ struct TestContext { { Stash stash; nbostream data; - encode_engine.encode(make_value(encode_engine, spec, stash), data, stash); - TEST_DO(verify_result(Eval::Result(decode_engine.decode(data, stash)), spec)); + encode_engine.encode(make_value(encode_engine, spec, stash), data); + TEST_DO(verify_result(Eval::Result(decode_engine, *decode_engine.decode(data)), spec)); } void verify_encode_decode(const TensorSpec &spec) { @@ -884,13 +882,13 @@ struct TestContext { const Inspector &binary = test["binary"]; EXPECT_GREATER(binary.entries(), 0u); nbostream encoded; - engine.encode(make_value(engine, spec, stash), encoded, stash); + engine.encode(make_value(engine, spec, stash), encoded); test.setData("encoded", Memory(encoded.peek(), encoded.size())); bool matched_encode = false; for (size_t i = 0; i < binary.entries(); ++i) { nbostream data = extract_data(binary[i].asString()); matched_encode = (matched_encode || is_same(encoded, data)); - TEST_DO(verify_result(Eval::Result(engine.decode(data, stash)), spec)); + TEST_DO(verify_result(Eval::Result(engine, *engine.decode(data)), spec)); EXPECT_EQUAL(data.size(), 0u); } EXPECT_TRUE(matched_encode); diff --git a/eval/src/vespa/eval/eval/value.cpp b/eval/src/vespa/eval/eval/value.cpp index 456d80c0ff0..4bfd758f9cd 100644 --- a/eval/src/vespa/eval/eval/value.cpp +++ b/eval/src/vespa/eval/eval/value.cpp @@ -6,19 +6,10 @@ namespace vespalib { namespace eval { -ErrorValue ErrorValue::instance; +ValueType ErrorValue::_type = ValueType::error_type(); +const ErrorValue ErrorValue::instance; -double -TensorValue::as_double() const -{ - return _tensor->as_double(); -} - -ValueType -TensorValue::type() const -{ - return _tensor->engine().type_of(*_tensor); -} +ValueType DoubleValue::_type = ValueType::double_type(); } // namespace vespalib::eval } // namespace vespalib diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 8826faed140..08ca9792739 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -5,7 +5,6 @@ #include <vespa/vespalib/stllike/string.h> #include <memory> #include <vespa/vespalib/util/stash.h> -#include "tensor.h" #include "value_type.h" namespace vespalib { @@ -25,43 +24,33 @@ struct Value { virtual bool is_double() const { return false; } virtual bool is_tensor() const { return false; } virtual double as_double() const { return 0.0; } - virtual bool as_bool() const { return false; } + bool as_bool() const { return (as_double() != 0.0); } virtual const Tensor *as_tensor() const { return nullptr; } - virtual ValueType type() const = 0; + virtual const ValueType &type() const = 0; virtual ~Value() {} }; -struct ErrorValue : public Value { - static ErrorValue instance; +class ErrorValue : public Value +{ +private: + static ValueType _type; +public: + static const ErrorValue instance; bool is_error() const override { return true; } double as_double() const override { return error_value; } - ValueType type() const override { return ValueType::error_type(); } + const ValueType &type() const override { return _type; } }; class DoubleValue : public Value { private: double _value; + static ValueType _type; public: DoubleValue(double value) : _value(value) {} bool is_double() const override { return true; } double as_double() const override { return _value; } - bool as_bool() const override { return (_value != 0.0); } - ValueType type() const override { return ValueType::double_type(); } -}; - -class TensorValue : public Value -{ -private: - const Tensor *_tensor; - std::unique_ptr<Tensor> _stored; -public: - TensorValue(const Tensor &value) : _tensor(&value), _stored() {} - TensorValue(std::unique_ptr<Tensor> value) : _tensor(value.get()), _stored(std::move(value)) {} - bool is_tensor() const override { return true; } - double as_double() const override; - const Tensor *as_tensor() const override { return _tensor; } - ValueType type() const override; + const ValueType &type() const override { return _type; } }; } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index f026ca060c6..38d5bbc643b 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -68,17 +68,13 @@ void decode_json(const vespalib::string &path, Slime &slime) { } // namespace vespalib::eval::<unnamed> -using ErrorConstant = SimpleConstantValue<ErrorValue>; -using TensorConstant = SimpleConstantValue<TensorValue>; - ConstantValue::UP ConstantTensorLoader::create(const vespalib::string &path, const vespalib::string &type) const { ValueType value_type = ValueType::from_spec(type); if (value_type.is_error()) { LOG(warning, "invalid type specification: %s", type.c_str()); - auto tensor = _engine.create(TensorSpec("double")); - return std::make_unique<TensorConstant>(_engine.type_of(*tensor), std::move(tensor)); + return std::make_unique<SimpleConstantValue>(_engine.from_spec(TensorSpec("double"))); } Slime slime; decode_json(path, slime); @@ -96,8 +92,7 @@ ConstantTensorLoader::create(const vespalib::string &path, const vespalib::strin cells[i]["address"].traverse(extractor); spec.add(address, cells[i]["value"].asDouble()); } - auto tensor = _engine.create(spec); - return std::make_unique<TensorConstant>(_engine.type_of(*tensor), std::move(tensor)); + return std::make_unique<SimpleConstantValue>(_engine.from_spec(spec)); } } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value_cache/constant_value.h b/eval/src/vespa/eval/eval/value_cache/constant_value.h index 462dc3ad9b4..ba7fe6fcf3d 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_value.h +++ b/eval/src/vespa/eval/eval/value_cache/constant_value.h @@ -21,19 +21,13 @@ struct ConstantValue { virtual ~ConstantValue() {} }; -/** - * A simple implementation of a constant value that bundles together a - * ValueType instance with a specific Value subclass instance. - **/ -template <typename VALUE> -struct SimpleConstantValue : ConstantValue { - ValueType my_type; - VALUE my_value; - template <typename... Args> - SimpleConstantValue(const ValueType &type_in, Args &&...args) - : my_type(type_in), my_value(std::forward<Args>(args)...) {} - const ValueType &type() const override { return my_type; } - const Value &value() const override { return my_value; } +class SimpleConstantValue : public ConstantValue { +private: + const Value::UP _value; +public: + SimpleConstantValue(Value::UP value) : _value(std::move(value)) {} + const ValueType &type() const override { return _value->type(); } + const Value &value() const override { return *_value; } }; /** diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 7adb95f69ca..2506e6fcf0e 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -11,6 +11,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/eval/operation.h> #include <cassert> @@ -21,8 +22,8 @@ using eval::Aggr; using eval::Aggregator; using eval::DoubleValue; using eval::ErrorValue; +using eval::TensorFunction; using eval::TensorSpec; -using eval::TensorValue; using eval::Value; using eval::ValueType; @@ -36,18 +37,13 @@ const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); // map tensors to simple tensors before fall-back evaluation -const eval::SimpleTensor &to_simple(const eval::Tensor &tensor, Stash &stash) { - if (auto wrapped = dynamic_cast<const WrappedSimpleTensor *>(&tensor)) { - return wrapped->get(); - } - TensorSpec spec = tensor.engine().to_spec(tensor); - using PTR = std::unique_ptr<eval::SimpleTensor>; - return *stash.create<PTR>(eval::SimpleTensor::create(spec)); -} - const Value &to_simple(const Value &value, Stash &stash) { if (auto tensor = value.as_tensor()) { - return stash.create<TensorValue>(to_simple(*tensor, stash)); + if (auto wrapped = dynamic_cast<const WrappedSimpleTensor *>(tensor)) { + return wrapped->get(); + } + TensorSpec spec = tensor->engine().to_spec(*tensor); + return *stash.create<Value::UP>(eval::SimpleTensor::create(spec)); } return value; } @@ -58,11 +54,11 @@ const Value &to_default(const Value &value, Stash &stash) { if (auto tensor = value.as_tensor()) { if (auto simple = dynamic_cast<const eval::SimpleTensor *>(tensor)) { if (!Tensor::supported({simple->type()})) { - return stash.create<TensorValue>(std::make_unique<WrappedSimpleTensor>(*simple)); + return stash.create<WrappedSimpleTensor>(*simple); } } TensorSpec spec = tensor->engine().to_spec(*tensor); - return stash.create<TensorValue>(default_engine().create(spec)); + return *stash.create<Value::UP>(default_engine().from_spec(spec)); } return value; } @@ -72,9 +68,19 @@ const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) { return ErrorValue::instance; } if (tensor->getType().is_tensor()) { - return stash.create<TensorValue>(std::move(tensor)); + return *stash.create<Value::UP>(std::move(tensor)); } - return stash.create<DoubleValue>(tensor->sum()); + return stash.create<DoubleValue>(tensor->as_double()); +} + +Value::UP to_value(std::unique_ptr<Tensor> tensor) { + if (!tensor) { + return std::make_unique<ErrorValue>(); + } + if (tensor->type().is_tensor()) { + return std::move(tensor); + } + return std::make_unique<DoubleValue>(tensor->as_double()); } const Value &fallback_join(const Value &a, const Value &b, join_fun_t function, Stash &stash) { @@ -89,38 +95,22 @@ const Value &fallback_reduce(const Value &a, eval::Aggr aggr, const std::vector< const DefaultTensorEngine DefaultTensorEngine::_engine; -eval::ValueType -DefaultTensorEngine::type_of(const Tensor &tensor) const -{ - assert(&tensor.engine() == this); - const tensor::Tensor &my_tensor = static_cast<const tensor::Tensor &>(tensor); - return my_tensor.getType(); -} - -vespalib::string -DefaultTensorEngine::to_string(const Tensor &tensor) const -{ - assert(&tensor.engine() == this); - const tensor::Tensor &my_tensor = static_cast<const tensor::Tensor &>(tensor); - return my_tensor.toString(); -} - TensorSpec -DefaultTensorEngine::to_spec(const Tensor &tensor) const -{ - assert(&tensor.engine() == this); - const tensor::Tensor &my_tensor = static_cast<const tensor::Tensor &>(tensor); - return my_tensor.toSpec(); -} - -eval::TensorFunction::UP -DefaultTensorEngine::compile(eval::tensor_function::Node_UP expr) const +DefaultTensorEngine::to_spec(const Value &value) const { - return DenseTensorFunctionCompiler::compile(std::move(expr)); + if (value.is_double()) { + return TensorSpec("double").add({}, value.as_double()); + } else if (auto tensor = value.as_tensor()) { + assert(&tensor->engine() == this); + const tensor::Tensor &my_tensor = static_cast<const tensor::Tensor &>(*tensor); + return my_tensor.toSpec(); + } else { + return TensorSpec("error"); + } } -std::unique_ptr<eval::Tensor> -DefaultTensorEngine::create(const TensorSpec &spec) const +Value::UP +DefaultTensorEngine::from_spec(const TensorSpec &spec) const { ValueType type = ValueType::from_spec(spec.type()); bool is_dense = false; @@ -149,7 +139,7 @@ DefaultTensorEngine::create(const TensorSpec &spec) const builder.addCell(cell.second); } return builder.build(); - } else { // sparse + } else if (is_sparse) { DefaultTensor::builder builder; std::map<vespalib::string,DefaultTensor::builder::Dimension> dimension_map; for (const auto &dimension: type.dimensions()) { @@ -163,6 +153,12 @@ DefaultTensorEngine::create(const TensorSpec &spec) const builder.add_cell(cell.second); } return builder.build(); + } else if (type.is_double()) { + double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; + return std::make_unique<DoubleValue>(value); + } else { + assert(type.is_error()); + return std::make_unique<ErrorValue>(); } } @@ -189,7 +185,7 @@ struct CellFunctionBindRightAdapter : tensor::CellFunction { //----------------------------------------------------------------------------- void -DefaultTensorEngine::encode(const Value &value, nbostream &output, Stash &) const +DefaultTensorEngine::encode(const Value &value, nbostream &output) const { if (auto tensor = value.as_tensor()) { TypedBinaryFormat::serialize(output, static_cast<const tensor::Tensor &>(*tensor)); @@ -198,10 +194,18 @@ DefaultTensorEngine::encode(const Value &value, nbostream &output, Stash &) cons } } -const Value & -DefaultTensorEngine::decode(nbostream &input, Stash &stash) const +Value::UP +DefaultTensorEngine::decode(nbostream &input) const +{ + return to_value(TypedBinaryFormat::deserialize(input)); +} + +//----------------------------------------------------------------------------- + +const TensorFunction & +DefaultTensorEngine::compile(const eval::tensor_function::Node &expr, Stash &stash) const { - return to_value(TypedBinaryFormat::deserialize(input), stash); + return DenseTensorFunctionCompiler::compile(expr, stash); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.h b/eval/src/vespa/eval/tensor/default_tensor_engine.h index bbb03aceb1f..1cef4ba2d35 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.h +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.h @@ -19,16 +19,14 @@ private: public: static const TensorEngine &ref() { return _engine; }; - ValueType type_of(const Tensor &tensor) const override; - vespalib::string to_string(const Tensor &tensor) const override; - TensorSpec to_spec(const Tensor &tensor) const override; + TensorSpec to_spec(const Value &value) const override; + Value::UP from_spec(const TensorSpec &spec) const override; - virtual eval::TensorFunction::UP compile(eval::tensor_function::Node_UP expr) const override; + void encode(const Value &value, nbostream &output) const override; + Value::UP decode(nbostream &input) const override; - std::unique_ptr<Tensor> create(const TensorSpec &spec) const override; + const TensorFunction &compile(const eval::tensor_function::Node &expr, Stash &stash) const override; - void encode(const Value &value, nbostream &output, Stash &stash) const override; - const Value &decode(nbostream &input, Stash &stash) const override; const Value &map(const Value &a, map_fun_t function, Stash &stash) const override; const Value &join(const Value &a, const Value &b, join_fun_t function, Stash &stash) const override; const Value &reduce(const Value &a, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) const override; diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index 530eaed9aa9..705496714fa 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -31,10 +31,10 @@ getCellsRef(const eval::Value &value) } const eval::Value & -DenseDotProductFunction::eval(const Input &input, Stash &stash) const +DenseDotProductFunction::eval(ConstArrayRef<eval::Value::CREF> params, Stash &stash) const { - DenseTensorView::CellsRef lhsCells = getCellsRef(input.get_tensor(_lhsTensorId)); - DenseTensorView::CellsRef rhsCells = getCellsRef(input.get_tensor(_rhsTensorId)); + DenseTensorView::CellsRef lhsCells = getCellsRef(params[_lhsTensorId]); + DenseTensorView::CellsRef rhsCells = getCellsRef(params[_rhsTensorId]); size_t numCells = std::min(lhsCells.size(), rhsCells.size()); double result = _hwAccelerator->dotProduct(lhsCells.cbegin(), rhsCells.cbegin(), numCells); return stash.create<eval::DoubleValue>(result); diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h index 905939cc781..8ad57d69524 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h @@ -24,7 +24,7 @@ public: DenseDotProductFunction(size_t lhsTensorId_, size_t rhsTensorId_); size_t lhsTensorId() const { return _lhsTensorId; } size_t rhsTensorId() const { return _rhsTensorId; } - virtual const eval::Value &eval(const Input &input, Stash &stash) const override; + const eval::Value &eval(ConstArrayRef<eval::Value::CREF> params, Stash &stash) const override; }; } // namespace tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp index 354c0a2f466..5d7e0c83267 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp @@ -26,7 +26,7 @@ calcCellsSize(const eval::ValueType &type) void checkCellsSize(const DenseTensor &arg) { - auto cellsSize = calcCellsSize(arg.type()); + auto cellsSize = calcCellsSize(arg.fast_type()); if (arg.cells().size() != cellsSize) { throw IllegalStateException(make_string("Wrong cell size, " "expected=%zu, " diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp index 10651b59468..65fee767690 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp @@ -14,8 +14,8 @@ template <typename Function> std::unique_ptr<Tensor> apply(const DenseTensorView &lhs, const DenseTensorView &rhs, Function &&func) { - DenseTensorAddressCombiner combiner(lhs.type(), rhs.type()); - DirectDenseTensorBuilder builder(DenseTensorAddressCombiner::combineDimensions(lhs.type(), rhs.type())); + DenseTensorAddressCombiner combiner(lhs.fast_type(), rhs.fast_type()); + DirectDenseTensorBuilder builder(DenseTensorAddressCombiner::combineDimensions(lhs.fast_type(), rhs.fast_type())); for (DenseTensorCellsIterator lhsItr = lhs.cellsIterator(); lhsItr.valid(); lhsItr.next()) { for (DenseTensorCellsIterator rhsItr = rhs.cellsIterator(); rhsItr.valid(); rhsItr.next()) { bool combineSuccess = combiner.combine(lhsItr, rhsItr); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h index 2d5257f018b..f77517bfdc5 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h @@ -34,7 +34,7 @@ public: void next(); double cell() const { return _cells[_cellIdx]; } const std::vector<size_t> &address() const { return _address; } - const eval::ValueType &type() const { return _type; } + const eval::ValueType &fast_type() const { return _type; } }; } // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp index a22307c25ad..e9ee7d30692 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp @@ -2,6 +2,7 @@ #include "dense_dot_product_function.h" #include "dense_tensor_function_compiler.h" +#include <vespa/eval/eval/operation.h> #include <vespa/vespalib/test/insertion_operators.h> #include <iostream> @@ -36,30 +37,30 @@ isCompatibleTensorsForDotProduct(const ValueType &lhsType, const ValueType &rhsT struct DotProductFunctionCompiler { - static TensorFunction::UP compile(Node_UP expr) { - const Reduce *reduce = as<Reduce>(*expr); + static const TensorFunction &compile(const Node &expr, Stash &stash) { + const Reduce *reduce = as<Reduce>(expr); if (reduce && (reduce->aggr == Aggr::SUM) && willReduceAllDimensions(reduce->dimensions)) { - const Join *join = as<Join>(*reduce->tensor); + const Join *join = as<Join>(reduce->tensor); if (join && (join->function == Mul::f)) { - const Inject *lhsTensor = as<Inject>(*join->lhs_tensor); - const Inject *rhsTensor = as<Inject>(*join->rhs_tensor); + const Inject *lhsTensor = as<Inject>(join->lhs_tensor); + const Inject *rhsTensor = as<Inject>(join->rhs_tensor); if (lhsTensor && rhsTensor && isCompatibleTensorsForDotProduct(lhsTensor->result_type, rhsTensor->result_type)) { - return std::make_unique<DenseDotProductFunction>(lhsTensor->tensor_id, rhsTensor->tensor_id); + return stash.create<DenseDotProductFunction>(lhsTensor->tensor_id, rhsTensor->tensor_id); } } } - return std::move(expr); + return expr; } }; } -TensorFunction::UP -DenseTensorFunctionCompiler::compile(Node_UP expr) +const TensorFunction & +DenseTensorFunctionCompiler::compile(const eval::tensor_function::Node &expr, Stash &stash) { - return DotProductFunctionCompiler::compile(std::move(expr)); + return DotProductFunctionCompiler::compile(expr, stash); } } // namespace tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h index ef940bf38f9..d5ba4e4f7a7 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h @@ -5,6 +5,9 @@ #include <vespa/eval/eval/tensor_function.h> namespace vespalib { + +class Stash; + namespace tensor { /** @@ -13,7 +16,7 @@ namespace tensor { */ struct DenseTensorFunctionCompiler { - static eval::TensorFunction::UP compile(eval::tensor_function::Node_UP expr); + static const eval::TensorFunction &compile(const eval::tensor_function::Node &expr, Stash &stash); }; } // namespace tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp index c6fc04bb27b..9f608921c05 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp @@ -94,7 +94,7 @@ template <typename Function> DenseTensor::UP reduce(const DenseTensorView &tensor, const vespalib::string &dimensionToRemove, Function &&func) { - DimensionReducer reducer(tensor.type(), dimensionToRemove); + DimensionReducer reducer(tensor.fast_type(), dimensionToRemove); return reducer.reduceCells(tensor.cellsRef(), func); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index 4f3e49f8ec1..4402b5b0ae0 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -49,7 +49,7 @@ calcCellsSize(const eval::ValueType &type) void checkCellsSize(const DenseTensorView &arg) { - auto cellsSize = calcCellsSize(arg.type()); + auto cellsSize = calcCellsSize(arg.fast_type()); if (arg.cellsRef().size() != cellsSize) { throw IllegalStateException(make_string("wrong cell size, " "expected=%zu, " @@ -63,14 +63,14 @@ void checkDimensions(const DenseTensorView &lhs, const DenseTensorView &rhs, vespalib::stringref operation) { - if (lhs.type() != rhs.type()) { + if (lhs.fast_type() != rhs.fast_type()) { throw IllegalStateException(make_string("mismatching dimensions for " "dense tensor %s, " "lhs dimensions = '%s', " "rhs dimensions = '%s'", operation.c_str(), - dimensionsAsString(lhs.type()).c_str(), - dimensionsAsString(rhs.type()).c_str())); + dimensionsAsString(lhs.fast_type()).c_str(), + dimensionsAsString(rhs.fast_type()).c_str())); } checkCellsSize(lhs); checkCellsSize(rhs); @@ -96,7 +96,7 @@ joinDenseTensors(const DenseTensorView &lhs, const DenseTensorView &rhs, ++rhsCellItr; } assert(rhsCellItr == rhs.cellsRef().cend()); - return std::make_unique<DenseTensor>(lhs.type(), + return std::make_unique<DenseTensor>(lhs.fast_type(), std::move(cells)); } @@ -132,7 +132,7 @@ bool sameCells(DenseTensorView::CellsRef lhs, DenseTensorView::CellsRef rhs) DenseTensorView::DenseTensorView(const DenseTensor &rhs) - : _typeRef(rhs.type()), + : _typeRef(rhs.fast_type()), _cellsRef(rhs.cellsRef()) { } @@ -260,7 +260,7 @@ void buildAddress(const DenseTensorCellsIterator &itr, TensorSpec::Address &address) { auto addressItr = itr.address().begin(); - for (const auto &dim : itr.type().dimensions()) { + for (const auto &dim : itr.fast_type().dimensions()) { address.emplace(std::make_pair(dim.name, TensorSpec::Label(*addressItr++))); } assert(addressItr == itr.address().end()); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index aa447eb42af..472cc58ad6b 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -42,7 +42,7 @@ public: : _typeRef(type_in), _cellsRef() {} - const eval::ValueType &type() const { return _typeRef; } + const eval::ValueType &fast_type() const { return _typeRef; } const CellsRef &cellsRef() const { return _cellsRef; } bool operator==(const DenseTensorView &rhs) const; CellsIterator cellsIterator() const { return CellsIterator(_typeRef, _cellsRef); } diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp index 5da7165af2a..71b7824ee5d 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp @@ -22,13 +22,13 @@ MutableDenseTensorView::MutableValueType::MutableValueType(ValueType type_in) MutableDenseTensorView::MutableValueType::~MutableValueType() {} MutableDenseTensorView::MutableDenseTensorView(ValueType type_in) - : DenseTensorView(_concreteType.type(), CellsRef()), + : DenseTensorView(_concreteType.fast_type(), CellsRef()), _concreteType(type_in) { } MutableDenseTensorView::MutableDenseTensorView(ValueType type_in, CellsRef cells_in) - : DenseTensorView(_concreteType.type(), cells_in), + : DenseTensorView(_concreteType.fast_type(), cells_in), _concreteType(type_in) { } diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h index e856885d0fa..7eee3a9483c 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h @@ -23,7 +23,7 @@ private: public: MutableValueType(eval::ValueType type_in); ~MutableValueType(); - const eval::ValueType &type() const { return _type; } + const eval::ValueType &fast_type() const { return _type; } void setUnboundDimensions(const uint32_t *unboundDimSizeBegin, const uint32_t *unboundDimSizeEnd) { const uint32_t *unboundDimSizePtr = unboundDimSizeBegin; for (auto unboundDimSize : _unboundDimSizes) { diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index bf522bcaed4..a2d600aa0c9 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -24,9 +24,9 @@ makeValueType(std::vector<eval::ValueType::Dimension> &&dimensions) { void DenseBinaryFormat::serialize(nbostream &stream, const DenseTensor &tensor) { - stream.putInt1_4Bytes(tensor.type().dimensions().size()); + stream.putInt1_4Bytes(tensor.fast_type().dimensions().size()); size_t cellsSize = 1; - for (const auto &dimension : tensor.type().dimensions()) { + for (const auto &dimension : tensor.fast_type().dimensions()) { stream.writeSmallString(dimension.name); stream.putInt1_4Bytes(dimension.size); cellsSize *= dimension.size; diff --git a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h index b0c75655ce5..33000d4889d 100644 --- a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h @@ -127,7 +127,7 @@ public: insertCell(address.getAddressRef(), value, [](double, double) -> double { abort(); }); } - eval::ValueType &type() { return _type; } + eval::ValueType &fast_type() { return _type; } Cells &cells() { return _cells; } }; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h index ad460f4849c..8f5f8066352 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h @@ -37,7 +37,7 @@ public: SparseTensor(eval::ValueType &&type_in, Cells &&cells_in, Stash &&stash_in); const Cells &cells() const { return _cells; } - const eval::ValueType &type() const { return _type; } + const eval::ValueType &fast_type() const { return _type; } bool operator==(const SparseTensor &rhs) const; eval::ValueType combineDimensionsWith(const SparseTensor &rhs) const; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp index cb22afc8fd5..4528c8ef1df 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp @@ -16,7 +16,7 @@ std::unique_ptr<Tensor> apply(const SparseTensor &lhs, const SparseTensor &rhs, Function &&func) { DirectTensorBuilder<SparseTensor> builder(lhs.combineDimensionsWith(rhs)); - TensorAddressCombiner addressCombiner(lhs.type(), rhs.type()); + TensorAddressCombiner addressCombiner(lhs.fast_type(), rhs.fast_type()); for (const auto &lhsCell : lhs.cells()) { for (const auto &rhsCell : rhs.cells()) { bool combineSuccess = addressCombiner.combine(lhsCell.first, diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp index 35ae6b7544b..b4c9d511d09 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp @@ -90,9 +90,9 @@ SparseTensorMatch::slowMatch(const TensorImplType &lhs, { std::vector<AddressOp> ops; SparseTensorAddressBuilder addressBuilder; - SparseTensorAddressPadder addressPadder(_builder.type(), - lhs.type()); - buildTransformOps(ops, lhs.type(), rhs.type()); + SparseTensorAddressPadder addressPadder(_builder.fast_type(), + lhs.fast_type()); + buildTransformOps(ops, lhs.fast_type(), rhs.fast_type()); for (const auto &lhsCell : lhs.cells()) { if (!transformAddress(addressBuilder, lhsCell.first, ops)) { continue; @@ -110,8 +110,8 @@ SparseTensorMatch::SparseTensorMatch(const TensorImplType &lhs, const TensorImplType &rhs) : Parent(lhs.combineDimensionsWith(rhs)) { - if ((lhs.type().dimensions().size() == rhs.type().dimensions().size()) && - (lhs.type().dimensions().size() == _builder.type().dimensions().size())) { + if ((lhs.fast_type().dimensions().size() == rhs.fast_type().dimensions().size()) && + (lhs.fast_type().dimensions().size() == _builder.fast_type().dimensions().size())) { // Ensure that first tensor to fastMatch has fewest cells. if (lhs.cells().size() <= rhs.cells().size()) { fastMatch(lhs, rhs); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp index 30b359eb73e..53ab8116255 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp @@ -45,11 +45,11 @@ reduce(const SparseTensor &tensor, if (dimensions.empty()) { return reduceAll(tensor, func); } - DirectTensorBuilder<SparseTensor> builder(tensor.type().reduce(dimensions)); - if (builder.type().dimensions().empty()) { + DirectTensorBuilder<SparseTensor> builder(tensor.fast_type().reduce(dimensions)); + if (builder.fast_type().dimensions().empty()) { return reduceAll(tensor, builder, func); } - TensorAddressReducer addressReducer(tensor.type(), dimensions); + TensorAddressReducer addressReducer(tensor.fast_type(), dimensions); for (const auto &cell : tensor.cells()) { addressReducer.reduce(cell.first); builder.insertCell(addressReducer.getAddressRef(), cell.second, func); diff --git a/eval/src/vespa/eval/tensor/tensor.h b/eval/src/vespa/eval/tensor/tensor.h index 3b3d7ce4a70..80afbbf52ff 100644 --- a/eval/src/vespa/eval/tensor/tensor.h +++ b/eval/src/vespa/eval/tensor/tensor.h @@ -31,6 +31,7 @@ struct Tensor : public eval::Tensor Tensor(); virtual ~Tensor() {} virtual const eval::ValueType &getType() const = 0; + virtual const eval::ValueType &type() const override { return getType(); } virtual double sum() const = 0; virtual double as_double() const final override { return sum(); } virtual Tensor::UP add(const Tensor &arg) const = 0; diff --git a/eval/src/vespa/eval/tensor/tensor_apply.cpp b/eval/src/vespa/eval/tensor/tensor_apply.cpp index f6ee7492b05..7c518d0516f 100644 --- a/eval/src/vespa/eval/tensor/tensor_apply.cpp +++ b/eval/src/vespa/eval/tensor/tensor_apply.cpp @@ -8,7 +8,7 @@ namespace tensor { template <class TensorT> TensorApply<TensorT>::TensorApply(const TensorImplType &tensor, const CellFunction &func) - : Parent(tensor.type()) + : Parent(tensor.fast_type()) { for (const auto &cell : tensor.cells()) { _builder.insertCell(cell.first, func.apply(cell.second)); diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.cpp b/eval/src/vespa/eval/tensor/tensor_mapper.cpp index 7c2c72abd46..25b369c246d 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.cpp +++ b/eval/src/vespa/eval/tensor/tensor_mapper.cpp @@ -69,7 +69,7 @@ mapAddress(const TensorAddress &address) { _addressBuilder.clear(); TensorAddressElementIterator<TensorAddress> addressIterator(address); - for (const auto &dimension : _builder.type().dimensions()) { + for (const auto &dimension : _builder.fast_type().dimensions()) { if (addressIterator.skipToDimension(dimension.name)) { _addressBuilder.add(addressIterator.label()); addressIterator.next(); diff --git a/eval/src/vespa/eval/tensor/tensor_operation.h b/eval/src/vespa/eval/tensor/tensor_operation.h index abf58641549..6975c21c448 100644 --- a/eval/src/vespa/eval/tensor/tensor_operation.h +++ b/eval/src/vespa/eval/tensor/tensor_operation.h @@ -28,17 +28,17 @@ protected: public: TensorOperation() : _builder(), - _type(_builder.type()), + _type(_builder.fast_type()), _cells(_builder.cells()) {} TensorOperation(const eval::ValueType &type) : _builder(type), - _type(_builder.type()), + _type(_builder.fast_type()), _cells(_builder.cells()) {} TensorOperation(const eval::ValueType &type, const Cells &cells) : _builder(type, cells), - _type(_builder.type()), + _type(_builder.fast_type()), _cells(_builder.cells()) {} Tensor::UP result() { diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index 534854732c7..7ad97a6e84e 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -20,7 +20,7 @@ WrappedSimpleTensor::equals(const Tensor &arg) const vespalib::string WrappedSimpleTensor::toString() const { - return eval::SimpleTensorEngine::ref().to_string(_tensor); + return toSpec().to_string(); } eval::TensorSpec diff --git a/fastlib/src/vespa/fastlib/net/httpserver.cpp b/fastlib/src/vespa/fastlib/net/httpserver.cpp index a9bba95a8ff..0d1b75ec7fe 100644 --- a/fastlib/src/vespa/fastlib/net/httpserver.cpp +++ b/fastlib/src/vespa/fastlib/net/httpserver.cpp @@ -367,7 +367,7 @@ int Fast_HTTPServer::Start(void) int retCode = FASTLIB_SUCCESS; { - std::unique_lock<std::mutex> runningGuard(_runningMutex); + std::lock_guard<std::mutex> runningGuard(_runningMutex); if (!_isRunning) { // Try listening retCode = Listen(); @@ -391,7 +391,7 @@ int Fast_HTTPServer::Start(void) void Fast_HTTPServer::Stop(void) { { - std::unique_lock<std::mutex> runningGuard(_runningMutex); + std::lock_guard<std::mutex> runningGuard(_runningMutex); _stopSignalled = true; if (_acceptThread) { _acceptThread->SetBreakFlag(); @@ -407,7 +407,7 @@ Fast_HTTPServer::Stop(void) { bool Fast_HTTPServer::StopSignalled(void) { bool retVal; - std::unique_lock<std::mutex> runningGuard(_runningMutex); + std::lock_guard<std::mutex> runningGuard(_runningMutex); retVal = _stopSignalled; return retVal; } @@ -458,7 +458,7 @@ void Fast_HTTPServer::Run(FastOS_ThreadInterface *thisThread, void *params) Fast_Socket *mySocket; { - std::unique_lock<std::mutex> runningGuard(_runningMutex); + std::lock_guard<std::mutex> runningGuard(_runningMutex); _isRunning = true; _stopSignalled = false; } @@ -516,7 +516,7 @@ void Fast_HTTPServer::Run(FastOS_ThreadInterface *thisThread, void *params) _serverSocket.SetSocketEvent(NULL); } - std::unique_lock<std::mutex> runningGuard(_runningMutex); + std::lock_guard<std::mutex> runningGuard(_runningMutex); _isRunning = false; } @@ -1040,7 +1040,7 @@ void Fast_HTTPServer::HandleFileRequest(const string & url, Fast_HTTPConnection& void Fast_HTTPServer::SetBaseDir(const char *baseDir) { - std::unique_lock<std::mutex> runningGuard(_runningMutex); + std::lock_guard<std::mutex> runningGuard(_runningMutex); if (!_isRunning) { _baseDir = baseDir; @@ -1178,14 +1178,14 @@ void Fast_HTTPServer::OutputNotFound(Fast_HTTPConnection& conn, void Fast_HTTPServer::AddConnection(Fast_HTTPConnection* connection) { - std::unique_lock<std::mutex> connectionGuard(_connectionLock); + std::lock_guard<std::mutex> connectionGuard(_connectionLock); _connections.Insert(connection); } void Fast_HTTPServer::RemoveConnection(Fast_HTTPConnection* connection) { - std::unique_lock<std::mutex> connectionGuard(_connectionLock); + std::lock_guard<std::mutex> connectionGuard(_connectionLock); _connections.RemoveElement(connection); _connectionCond.notify_one(); } diff --git a/fastlib/src/vespa/fastlib/text/normwordfolder.cpp b/fastlib/src/vespa/fastlib/text/normwordfolder.cpp index f383ff85df5..ca1f260515f 100644 --- a/fastlib/src/vespa/fastlib/text/normwordfolder.cpp +++ b/fastlib/src/vespa/fastlib/text/normwordfolder.cpp @@ -29,7 +29,7 @@ Fast_NormalizeWordFolder::Setup(uint32_t flags) { // Only allow setting these when not initialized or initializing... { - std::unique_lock<std::mutex> initGuard(_initMutex); + std::lock_guard<std::mutex> initGuard(_initMutex); _doAccentRemoval = (DO_ACCENT_REMOVAL & flags) != 0; // _doSmallToNormalKana = (DO_SMALL_TO_NORMAL_KANA & flags) != 0; // _doKatakanaToHiragana = (DO_KATAKANA_TO_HIRAGANA & flags) != 0; @@ -48,7 +48,7 @@ Fast_NormalizeWordFolder::Initialize() { unsigned int i; if (!_isInitialized) { - std::unique_lock<std::mutex> initGuard(_initMutex); + std::lock_guard<std::mutex> initGuard(_initMutex); if (!_isInitialized) { for (i = 0; i < 128; i++) diff --git a/fileacquirer/pom.xml b/fileacquirer/pom.xml index 4a7d22a10f5..eb040eddffb 100644 --- a/fileacquirer/pom.xml +++ b/fileacquirer/pom.xml @@ -21,11 +21,6 @@ <artifactId>config</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>filedistribution</artifactId> - <version>${project.version}</version> - </dependency> </dependencies> <build> <plugins> diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 0a4b75788bc..10b77b540e7 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -3,44 +3,66 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> - </parent> - <artifactId>filedistribution</artifactId> - <version>6-SNAPSHOT</version> - <packaging>jar</packaging> - <name>${project.artifactId}</name> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> + <modelVersion>4.0.0</modelVersion> + + <parent> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-class-plugin</artifactId> - <version>${project.version}</version> - <configuration> - <defFilesDirectories> - src/vespa/filedistribution/distributor, - src/vespa/filedistribution/model - </defFilesDirectories> - </configuration> - <executions> - <execution> - <id>config-gen</id> - <goals> - <goal>config-gen</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + + <artifactId>filedistribution</artifactId> + <version>6-SNAPSHOT</version> + <packaging>container-plugin</packaging> + <name>${project.artifactId}</name> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jrt</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + </plugins> + </build> + </project> diff --git a/filedistribution/src/apps/filedistributor/filedistributor.cpp b/filedistribution/src/apps/filedistributor/filedistributor.cpp index be15d23f27c..f2bf5fa0ccd 100644 --- a/filedistribution/src/apps/filedistributor/filedistributor.cpp +++ b/filedistribution/src/apps/filedistributor/filedistributor.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/filedistribution/distributor/config-filedistributor.h> -#include <vespa/filedistribution/model/config-filereferences.h> +#include <vespa/config-filedistributor.h> +#include <vespa/config-filereferences.h> #include <vespa/filedistribution/distributor/filedistributortrackerimpl.h> #include <vespa/filedistribution/distributor/filedownloadermanager.h> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java new file mode 100644 index 00000000000..d09cf17b9e3 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java @@ -0,0 +1,127 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.jrt.DoubleArray; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringArray; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.log.LogLevel; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * An RPC server that handles file distribution requests. + * + * @author hmusum + */ +public class FileDistributionRpcServer { + + private final static Logger log = Logger.getLogger(FileDistributionRpcServer.class.getName()); + + private final Supervisor supervisor; + private final FileDownloader downloader; + + public FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { + this.supervisor = supervisor; + this.downloader = downloader; + declareFileDistributionMethods(); + } + + private void declareFileDistributionMethods() { + // Legacy method, needs to be the same name as used in filedistributor + supervisor.addMethod(new Method("waitFor", "s", "s", + this, "getFile") + .methodDesc("get path to file reference") + .paramDesc(0, "file reference", "file reference") + .returnDesc(0, "path", "path to file")); + supervisor.addMethod(new Method("filedistribution.getFile", "s", "s", + this, "getFile") + .methodDesc("get path to file reference") + .paramDesc(0, "file reference", "file reference") + .returnDesc(0, "path", "path to file")); + supervisor.addMethod(new Method("filedistribution.getActiveFileReferencesStatus", "", "SD", + this, "getActiveFileReferencesStatus") + .methodDesc("download status for file references") + .returnDesc(0, "file references", "array of file references") + .returnDesc(1, "download status", "percentage downloaded of each file reference in above array")); + supervisor.addMethod(new Method("filedistribution.setFileReferencesToDownload", "S", "i", + this, "setFileReferencesToDownload") + .methodDesc("set which file references to download") + .paramDesc(0, "file references", "file reference to download") + .returnDesc(0, "ret", "0 if success, 1 otherwise")); + } + + + //---------------- RPC methods ------------------------------------ + // TODO: Duplicate of code in FileAcquirereImpl. Find out where to put it. What about C++ code using this RPC call? + private static final int baseErrorCode = 0x10000; + private static final int baseFileProviderErrorCode = baseErrorCode + 0x1000; + + private static final int fileReferenceDoesNotExists = baseFileProviderErrorCode; + private static final int fileReferenceRemoved = fileReferenceDoesNotExists + 1; + private static final int fileReferenceInternalError = fileReferenceRemoved + 1; + + @SuppressWarnings({"UnusedDeclaration"}) + public final void getFile(Request req) { + req.detach(); + FileReference fileReference = new FileReference(req.parameters().get(0).asString()); + log.log(LogLevel.DEBUG, "getFile() called for file reference '" + fileReference.value() + "'"); + Optional<File> pathToFile = downloader.getFile(fileReference); + try { + if (pathToFile.isPresent()) { + req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); + } else { + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); + req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); + } + } catch (Throwable e) { + log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exeption: " + e.getMessage()); + req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); + } + req.returnRequest(); + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void getActiveFileReferencesStatus(Request req) { + Map<FileReference, Double> downloadStatus = downloader.downloadStatus(); + + String[] fileRefArray = new String[downloadStatus.keySet().size()]; + fileRefArray = downloadStatus.keySet().stream() + .map(FileReference::value) + .collect(Collectors.toList()) + .toArray(fileRefArray); + + double[] downloadStatusArray = new double[downloadStatus.values().size()]; + int i = 0; + for (Double d : downloadStatus.values()) { + downloadStatusArray[i++] = d; + } + + req.returnValues().add(new StringArray(fileRefArray)); + req.returnValues().add(new DoubleArray(downloadStatusArray)); + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void setFileReferencesToDownload(Request req) { + String[] fileReferenceStrings = req.parameters().get(0).asStringArray(); + List<FileReference> fileReferences = Stream.of(fileReferenceStrings) + .map(FileReference::new) + .collect(Collectors.toList()); + downloader.queueForDownload(fileReferences); + + req.returnValues().add(new Int32Value(0)); + } + + +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java new file mode 100644 index 00000000000..fde410bc8d7 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -0,0 +1,152 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.yahoo.config.FileReference; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; + +import java.io.File; +import java.time.Duration; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; + +/** + * Handles downloads of files (file references only for now) + * + * @author hmusum + */ +public class FileDownloader { + + private final static Logger log = Logger.getLogger(FileDownloader.class.getName()); + + private final File downloadDirectory; + private final Duration timeout; + private final FileReferenceDownloader fileReferenceDownloader; + + public FileDownloader(ConnectionPool connectionPool) { + this(connectionPool, + new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")), + Duration.ofMinutes(15)); + } + + FileDownloader(ConnectionPool connectionPool, File downloadDirectory, Duration timeout) { + this.downloadDirectory = downloadDirectory; + this.timeout = timeout; + this.fileReferenceDownloader = new FileReferenceDownloader(downloadDirectory, connectionPool, timeout); + } + + public Optional<File> getFile(FileReference fileReference) { + Objects.requireNonNull(fileReference, "file reference cannot be null"); + File directory = new File(downloadDirectory, fileReference.value()); + log.log(LogLevel.DEBUG, "Checking if there is a file in '" + directory.getAbsolutePath() + "' "); + + Optional<File> file = getFileFromFileSystem(fileReference, directory); + if (file.isPresent()) { + return file; + } else { + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found in " + + directory.getAbsolutePath() + ", starting download"); + return queueForDownload(fileReference, timeout); + } + } + + public void queueForDownload(List<FileReference> fileReferences) { + fileReferences.forEach(this::queueForDownload); + } + + public void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { + fileReferenceDownloader.receiveFile(fileReference, filename, content, xxHash); + } + + double downloadStatus(FileReference fileReference) { + return fileReferenceDownloader.downloadStatus(fileReference.value()); + } + + public Map<FileReference, Double> downloadStatus() { + return fileReferenceDownloader.downloadStatus(); + } + + File downloadDirectory() { + return downloadDirectory; + } + + private Optional<File> getFileFromFileSystem(FileReference fileReference, File directory) { + File[] files = directory.listFiles(); + if (directory.exists() && directory.isDirectory() && files != null && files.length > 0) { + if (files.length != 1) { + throw new RuntimeException("More than one file in '" + fileReference.value() + + "', expected only one, unable to proceed"); + } + File file = files[0]; + if (!file.exists()) { + throw new RuntimeException("File with reference '" + fileReference.value() + + "' does not exist"); + } else if (!file.canRead()) { + throw new RuntimeException("File with reference '" + fileReference.value() + + "'exists, but unable to read it"); + } else { + fileReferenceDownloader.setDownloadStatus(fileReference.value(), 100.0); + return Optional.of(file); + } + } + return Optional.empty(); + } + + private synchronized Optional<File> queueForDownload(FileReference fileReference, Duration timeout) { + if (fileReferenceDownloader.isDownloading(fileReference)) { + log.log(LogLevel.INFO, "Already downloading '" + fileReference.value() + "'"); + ListenableFuture<Optional<File>> future = + fileReferenceDownloader.addDownloadListener(fileReference, () -> getFile(fileReference)); + try { + return future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Failed downloading file reference '" + fileReference.value() + "': " + + Exceptions.toMessageString(e)); + } + } + + SettableFuture<Optional<File>> future = SettableFuture.create(); + queueForDownload(new FileReferenceDownload(fileReference, future)); + log.log(LogLevel.INFO, "Queued '" + fileReference.value() + "' for download with timeout " + timeout); + + try { + Optional<File> fileDownloaded; + try { + log.log(LogLevel.INFO, "Waiting for '" + fileReference.value() + "' to download"); + fileDownloaded = future.get(timeout.getSeconds() - 1, TimeUnit.SECONDS); + log.log(LogLevel.INFO, "'" + fileReference.value() + "' downloaded"); + } catch (TimeoutException e) { + log.log(LogLevel.WARNING, "Downloading '" + fileReference.value() + "' timed out"); + return Optional.empty(); + } + return fileDownloaded; + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException("Could not download '" + fileReference.value() + "'"); + } + } + + // We don't care about the future in this call + private synchronized void queueForDownload(FileReference fileReference) { + queueForDownload(new FileReferenceDownload(fileReference, SettableFuture.create())); + } + + private synchronized void queueForDownload(FileReferenceDownload fileReferenceDownload) { + fileReferenceDownloader.addToDownloadQueue(fileReferenceDownload); + } + + public FileReferenceDownloader fileReferenceDownloader() { + return fileReferenceDownloader; + } +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java new file mode 100644 index 00000000000..3999389d2c7 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -0,0 +1,133 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Supervisor; +import com.yahoo.log.LogLevel; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +public class FileReceiver { + + private final static Logger log = Logger.getLogger(FileReceiver.class.getName()); + private final static String RECEIVE_METHOD = "filedistribution.receiveFile"; + private final static String RECEIVE_META_METHOD = "filedistribution.receiveFileMeta"; + private final static String RECEIVE_PART_METHOD = "filedistribution.receiveFilePart"; + private final static String RECEIVE_EOF_METHOD = "filedistribution.receiveFileEof"; + + private final Supervisor supervisor; + private final FileReferenceDownloader downloader; + private final File downloadDirectory; + private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + + public FileReceiver(Supervisor supervisor, FileReferenceDownloader downloader, File downloadDirectory) { + this.supervisor = supervisor; + this.downloader = downloader; + this.downloadDirectory = downloadDirectory; + registerMethods(); + } + + private void registerMethods() { + receiveFileMethod(this).forEach((method) -> supervisor.addMethod(method)); + } + + // Defined here so that it can be added to supervisor used by client (server will use same connection when calling + // receiveFile after getting a serveFile method call). handler needs to implement receiveFile method + private List<Method> receiveFileMethod(Object handler) { + List<Method> methods = new ArrayList<>(); + methods.add(new Method(RECEIVE_META_METHOD, "ssl", "ii", handler,"receiveFileMeta") + .paramDesc(0, "filereference", "file reference to download") + .paramDesc(1, "filename", "filename") + .paramDesc(2, "filelength", "length in bytes of file") + .returnDesc(0, "ret", "0 if success, 1 otherwise") + .returnDesc(1, "session-id", "Session id to be used for this transfer")); + methods.add(new Method(RECEIVE_PART_METHOD, "siix", "i", handler,"receiveFilePart") + .paramDesc(0, "filereference", "file reference to download") + .paramDesc(1, "session-id", "Session id to be used for this transfer") + .paramDesc(2, "partid", "relative part number starting at zero") + .paramDesc(3, "data", "bytes in this part") + .returnDesc(0, "ret", "0 if success, 1 otherwise")); + methods.add(new Method(RECEIVE_EOF_METHOD, "silis", "i", handler,"receiveFileEof") + .paramDesc(0, "filereference", "file reference to download") + .paramDesc(1, "session-id", "Session id to be used for this transfer") + .paramDesc(2, "crc-code", "crc code (xxhash64)") + .paramDesc(3, "error-code", "Error code. 0 if none") + .paramDesc(4, "error-description", "Error description.") + .returnDesc(0, "ret", "0 if success, 1 if crc mismatch, 2 otherwise")); + // Temporary method until we have chunking + methods.add(new Method(RECEIVE_METHOD, "ssxlis", "i", handler, "receiveFile") + .methodDesc("receive file reference content") + .paramDesc(0, "file reference", "file reference to download") + .paramDesc(1, "filename", "filename") + .paramDesc(2, "content", "array of bytes") + .paramDesc(3, "hash", "xx64hash of the file content") + .paramDesc(4, "errorcode", "Error code. 0 if none") + .paramDesc(5, "error-description", "Error description.") + .returnDesc(0, "ret", "0 if success, 1 otherwise")); + return methods; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void receiveFile(Request req) { + FileReference fileReference = new FileReference(req.parameters().get(0).asString()); + String filename = req.parameters().get(1).asString(); + byte[] content = req.parameters().get(2).asData(); + long xxhash = req.parameters().get(3).asInt64(); + int errorCode = req.parameters().get(4).asInt32(); + String errorDescription = req.parameters().get(5).asString(); + + if (errorCode == 0) { + // TODO: Remove when system test works + log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'"); + receiveFile(fileReference, filename, content, xxhash); + req.returnValues().add(new Int32Value(0)); + } else { + log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription); + req.returnValues().add(new Int32Value(1)); + // TODO: Add error description return value here too? + } + } + + void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { + long xxHashFromContent = hasher.hash(ByteBuffer.wrap(content), 0); + if (xxHashFromContent != xxHash) + throw new RuntimeException("xxhash from content (" + xxHashFromContent + ") is not equal to xxhash in request (" + xxHash + ")"); + + File fileReferenceDir = new File(downloadDirectory, fileReference.value()); + try { + Files.createDirectories(fileReferenceDir.toPath()); + File file = new File(fileReferenceDir, filename); + log.log(LogLevel.INFO, "Writing data to " + file.getAbsolutePath()); + Files.write(file.toPath(), content); + downloader.completedDownloading(fileReference, file); + } catch (IOException e) { + log.log(LogLevel.ERROR, "Failed writing file: " + e.getMessage()); + throw new RuntimeException("Failed writing file: ", e); + } + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void receiveFileMeta(Request req) { + log.info("Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + } + @SuppressWarnings({"UnusedDeclaration"}) + public final void receiveFilePart(Request req) { + log.info("Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + } + @SuppressWarnings({"UnusedDeclaration"}) + public final void receiveFileEof(Request req) { + log.info("Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + } +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java new file mode 100644 index 00000000000..fb511411128 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution; + +import com.google.common.util.concurrent.SettableFuture; +import com.yahoo.config.FileReference; + +import java.io.File; +import java.util.Optional; + +public class FileReferenceDownload { + private final FileReference fileReference; + private final SettableFuture<Optional<File>> future; + + FileReferenceDownload(FileReference fileReference, SettableFuture<Optional<File>> future) { + this.fileReference = fileReference; + this.future = future; + } + + FileReference fileReference() { + return fileReference; + } + + SettableFuture<Optional<File>> future() { + return future; + } + +} diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java new file mode 100644 index 00000000000..4c9c37dd6da --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -0,0 +1,190 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.FileReference; +import com.yahoo.jrt.ErrorCode; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; + +import java.io.File; +import java.time.Duration; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Downloads file reference using rpc requests to config server and keeps track of files being downloaded + * <p> + * Some methods are synchronized to make sure access to downloads is atomic + * + * @author hmusum + */ +// TODO: Handle shutdown of executors +public class FileReferenceDownloader { + + private final static Logger log = Logger.getLogger(FileReferenceDownloader.class.getName()); + private final static Duration rpcTimeout = Duration.ofSeconds(10); + + private final ExecutorService downloadExecutor = + Executors.newFixedThreadPool(10, new DaemonThreadFactory("filereference downloader")); + private ExecutorService readFromQueueExecutor = + Executors.newFixedThreadPool(1, new DaemonThreadFactory("filereference download queue")); + private final ConnectionPool connectionPool; + private final ConcurrentLinkedQueue<FileReferenceDownload> downloadQueue = new ConcurrentLinkedQueue<>(); + private final Map<FileReference, FileReferenceDownload> downloads = new LinkedHashMap<>(); + private final Map<FileReference, Double> downloadStatus = new HashMap<>(); + private final Duration downloadTimeout; + private final FileReceiver fileReceiver; + + FileReferenceDownloader(File downloadDirectory, ConnectionPool connectionPool, Duration timeout) { + this.connectionPool = connectionPool; + this.downloadTimeout = timeout; + readFromQueueExecutor.submit(this::readFromQueue); + this.fileReceiver = new FileReceiver(connectionPool.getSupervisor(), this, downloadDirectory); + } + + private synchronized Optional<File> startDownload(FileReference fileReference, + Duration timeout, + FileReferenceDownload fileReferenceDownload) + throws ExecutionException, InterruptedException, TimeoutException { + downloads.put(fileReference, fileReferenceDownload); + setDownloadStatus(fileReference.value(), 0.0); + + int numAttempts = 0; + boolean downloadStarted = false; + do { + if (startDownloadRpc(fileReference)) + downloadStarted = true; + else + Thread.sleep(100); + } while (!downloadStarted && ++numAttempts <= 10); // TODO: How long/many times to retry? + + if (downloadStarted) { + return fileReferenceDownload.future().get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } else { + fileReferenceDownload.future().setException(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'")); + downloads.remove(fileReference); + return Optional.empty(); + } + } + + synchronized void addToDownloadQueue(FileReferenceDownload fileReferenceDownload) { + downloadQueue.add(fileReferenceDownload); + } + + void receiveFile(FileReference fileReference, String filename, byte[] content, long xxHash) { + fileReceiver.receiveFile(fileReference, filename, content, xxHash); + } + + synchronized Set<FileReference> queuedDownloads() { + return downloadQueue.stream() + .map(FileReferenceDownload::fileReference) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + private void readFromQueue() { + do { + FileReferenceDownload fileReferenceDownload = downloadQueue.poll(); + if (fileReferenceDownload == null) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { /* ignore for now */} + } else { + log.log(LogLevel.DEBUG, "Will download file reference '" + fileReferenceDownload.fileReference().value() + "'"); + downloadExecutor.submit(() -> startDownload(fileReferenceDownload.fileReference(), downloadTimeout, fileReferenceDownload)); + } + } while (true); + } + + void completedDownloading(FileReference fileReference, File file) { + if (downloads.containsKey(fileReference)) + downloads.get(fileReference).future().set(Optional.of(file)); + downloadStatus.put(fileReference, 100.0); + } + + private boolean startDownloadRpc(FileReference fileReference) throws ExecutionException, InterruptedException { + Connection connection = connectionPool.getCurrent(); + Request request = new Request("filedistribution.serveFile"); + request.parameters().add(new StringValue(fileReference.value())); + + execute(request, connection); + if (validateResponse(request)) { + log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + request + "\nSpec: " + connection); + if (request.returnValues().get(0).asInt32() == 0) { + log.log(LogLevel.INFO, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress()); + return true; + } else { + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress()); + connectionPool.setNewCurrentConnection(); + return false; + } + } else { + log.log(LogLevel.WARNING, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() + + ", error code: " + request.errorCode()); + if (request.isError() && request.errorCode() == ErrorCode.CONNECTION || request.errorCode() == ErrorCode.TIMEOUT) { + log.log(LogLevel.WARNING, "Setting error for connection " + connection.getAddress()); + connectionPool.setError(connection, request.errorCode()); + } + return false; + } + } + + synchronized boolean isDownloading(FileReference fileReference) { + return downloads.containsKey(fileReference); + } + + synchronized ListenableFuture<Optional<File>> addDownloadListener(FileReference fileReference, Runnable runnable) { + FileReferenceDownload fileReferenceDownload = downloads.get(fileReference); + fileReferenceDownload.future().addListener(runnable, downloadExecutor); + return fileReferenceDownload.future(); + } + + private void execute(Request request, Connection connection) { + connection.invokeSync(request, (double) rpcTimeout.getSeconds()); + } + + private boolean validateResponse(Request request) { + if (request.isError()) { + return false; + } else if (request.returnValues().size() == 0) { + return false; + } else if (!request.checkReturnTypes("is")) { // TODO: Do not hard-code return type + log.log(LogLevel.WARNING, "Invalid return types for response: " + request.errorMessage()); + return false; + } + return true; + } + + double downloadStatus(String file) { + return downloadStatus.getOrDefault(new FileReference(file), 0.0); + } + + void setDownloadStatus(String file, double percentageDownloaded) { + downloadStatus.put(new FileReference(file), percentageDownloaded); + } + + Map<FileReference, Double> downloadStatus() { + return ImmutableMap.copyOf(downloadStatus); + } + + public ConnectionPool connectionPool() { + return connectionPool; + } +} diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java new file mode 100644 index 00000000000..278c46dab8b --- /dev/null +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -0,0 +1,335 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; +import net.jpountz.xxhash.XXHash64; +import net.jpountz.xxhash.XXHashFactory; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.jrt.ErrorCode.CONNECTION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FileDownloaderTest { + + private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); + + private MockConnection connection; + private FileDownloader fileDownloader; + private File downloadDir; + + @Before + public void setup() { + try { + downloadDir = Files.createTempDirectory("filedistribution").toFile(); + connection = new MockConnection(); + fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(2000)); + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void getFile() throws IOException { + File downloadDir = fileDownloader.downloadDirectory(); + + { + // fileReference already exists on disk, does not have to be downloaded + + String fileReferenceString = "foo"; + String filename = "foo.jar"; + FileReference fileReference = new FileReference(fileReferenceString); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + writeFileReference(downloadDir, fileReferenceString, filename); + + // Check that we get correct path and content when asking for file reference + Optional<File> pathToFile = fileDownloader.getFile(fileReference); + assertTrue(pathToFile.isPresent()); + String downloadedFile = new File(fileReferenceFullPath, filename).getAbsolutePath(); + assertEquals(new File(fileReferenceFullPath, filename).getAbsolutePath(), downloadedFile); + assertEquals("content", IOUtils.readFile(pathToFile.get())); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + } + + { + // fileReference does not exist on disk, needs to be downloaded, but fails when asking upstream for file) + + connection.setResponseHandler(new MockConnection.UnknownFileReferenceResponseHandler()); + + FileReference fileReference = new FileReference("bar"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status when unable to download + assertDownloadStatus(fileDownloader, fileReference, 0.0); + } + + { + // fileReference does not exist on disk, needs to be downloaded) + + FileReference fileReference = new FileReference("fileReference"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status + assertDownloadStatus(fileDownloader, fileReference, 0.0); + + // Receives fileReference, should return and make it available to caller + String filename = "abc.jar"; + receiveFile(fileReference, filename, "some other content"); + Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + + assertTrue(downloadedFile.isPresent()); + File downloadedFileFullPath = new File(fileReferenceFullPath, filename); + assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath()); + assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + } + } + + @Test + public void getFileWhenConnectionError() throws IOException { + fileDownloader = new FileDownloader(connection, downloadDir, Duration.ofMillis(3000)); + File downloadDir = fileDownloader.downloadDirectory(); + + int timesToFail = 2; + MockConnection.ConnectionErrorResponseHandler responseHandler = new MockConnection.ConnectionErrorResponseHandler(timesToFail); + connection.setResponseHandler(responseHandler); + + FileReference fileReference = new FileReference("fileReference"); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); + assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + + // Verify download status + assertDownloadStatus(fileDownloader, fileReference, 0.0); + + // Receives fileReference, should return and make it available to caller + String filename = "abc.jar"; + receiveFile(fileReference, filename, "some other content"); + Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + + assertTrue(downloadedFile.isPresent()); + File downloadedFileFullPath = new File(fileReferenceFullPath, filename); + assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath()); + assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); + + // Verify download status when downloaded + assertDownloadStatus(fileDownloader, fileReference, 100.0); + + assertEquals(timesToFail, responseHandler.failedTimes); + } + + @Test + public void setFilesToDownload() throws IOException { + Duration timeout = Duration.ofMillis(200); + File downloadDir = Files.createTempDirectory("filedistribution").toFile(); + MockConnection connectionPool = new MockConnection(); + connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); + FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, timeout); + FileReference foo = new FileReference("foo"); + FileReference bar = new FileReference("bar"); + List<FileReference> fileReferences = Arrays.asList(foo, bar); + fileDownloader.queueForDownload(fileReferences); + + // Verify download status + assertDownloadStatus(fileDownloader, foo, 0.0); + assertDownloadStatus(fileDownloader, bar, 0.0); + } + + @Test + public void receiveFile() throws IOException { + FileReference foo = new FileReference("foo"); + String filename = "foo.jar"; + receiveFile(foo, filename, "content"); + File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename); + assertEquals("content", IOUtils.readFile(downloadedFile)); + } + + private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { + File file = new File(new File(dir, fileReferenceString), fileName); + IOUtils.writeFile(file, "content", false); + } + + private File fileReferenceFullPath(File dir, FileReference fileReference) { + return new File(dir, fileReference.value()); + } + + private void assertDownloadStatus(FileDownloader fileDownloader, FileReference fileReference, double expectedDownloadStatus) { + double downloadStatus = fileDownloader.downloadStatus(fileReference); + assertEquals(expectedDownloadStatus, downloadStatus, 0.0001); + } + + private void receiveFile(FileReference fileReference, String filename, String content) { + byte[] contentBytes = Utf8.toBytes(content); + long xxHashFromContent = hasher.hash(ByteBuffer.wrap(contentBytes), 0); + fileDownloader.receiveFile(fileReference, filename, contentBytes, xxHashFromContent); + } + + private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { + + private ResponseHandler responseHandler; + + MockConnection() { + this(new FileReferenceFoundResponseHandler()); + } + + MockConnection(ResponseHandler responseHandler) { + this.responseHandler = responseHandler; + } + + @Override + public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { + responseHandler.request(request); + } + + @Override + public void invokeSync(Request request, double jrtTimeout) { + responseHandler.request(request); + } + + @Override + public void setError(int errorCode) { + } + + @Override + public void setSuccess() { + } + + @Override + public String getAddress() { + return null; + } + + @Override + public void close() { + } + + @Override + public void setError(Connection connection, int errorCode) { + connection.setError(errorCode); + } + + @Override + public Connection getCurrent() { + return this; + } + + @Override + public Connection setNewCurrentConnection() { + return this; + } + + @Override + public int getSize() { + return 1; + } + + @Override + public Supervisor getSupervisor() { + return new Supervisor(new Transport()); + } + + void setResponseHandler(ResponseHandler responseHandler) { + this.responseHandler = responseHandler; + } + + public interface ResponseHandler { + void request(Request request); + } + + static class FileReferenceFoundResponseHandler implements MockConnection.ResponseHandler { + + @Override + public void request(Request request) { + if (request.methodName().equals("filedistribution.serveFile")) { + request.returnValues().add(new Int32Value(0)); + request.returnValues().add(new StringValue("OK")); + } + } + } + + static class UnknownFileReferenceResponseHandler implements MockConnection.ResponseHandler { + + @Override + public void request(Request request) { + if (request.methodName().equals("filedistribution.serveFile")) { + request.returnValues().add(new Int32Value(1)); + request.returnValues().add(new StringValue("Internal error")); + } + } + } + + static class WaitResponseHandler implements MockConnection.ResponseHandler { + + private final Duration waitUntilAnswering; + + WaitResponseHandler(Duration waitUntilAnswering) { + super(); + this.waitUntilAnswering = waitUntilAnswering; + } + + @Override + public void request(Request request) { + try { Thread.sleep(waitUntilAnswering.toMillis());} catch (InterruptedException e) { /* do nothing */ } + + if (request.methodName().equals("filedistribution.serveFile")) { + request.returnValues().add(new Int32Value(0)); + request.returnValues().add(new StringValue("OK")); + } + } + } + + static class ConnectionErrorResponseHandler implements MockConnection.ResponseHandler { + + private final int timesToFail; + private int failedTimes = 0; + + ConnectionErrorResponseHandler(int timesToFail) { + super(); + this.timesToFail = timesToFail; + } + + @Override + public void request(Request request) { + if (request.methodName().equals("filedistribution.serveFile")) { + if (failedTimes < timesToFail) { + request.setError(CONNECTION, "Connection error"); + failedTimes++; + } else { + request.returnValues().add(new Int32Value(0)); + request.returnValues().add(new StringValue("OK")); + } + } + } + } + } + +} diff --git a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt index f85ab85fb39..6b575dc5526 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt @@ -11,5 +11,3 @@ vespa_add_library(filedistribution_distributor STATIC DEPENDS ) target_compile_options(filedistribution_distributor PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) -vespa_generate_config(filedistribution_distributor filedistributor.def) -install_config_definition(filedistributor.def cloud.config.filedistribution.filedistributor.def) diff --git a/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp b/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp index 954cce23205..f0825343f27 100644 --- a/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp +++ b/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp @@ -174,20 +174,6 @@ Java_com_yahoo_vespa_filedistribution_FileDistributionManager_setDeployedFilesIm JNIEXPORT void JNICALL -Java_com_yahoo_vespa_filedistribution_FileDistributionManager_limitSendingOfDeployedFilesToImpl( - JNIEnv *env, jobject self, jobjectArray hostNamesArg, jbyteArray appIdArg) -{ - try { - JNIArray<JNIString> hostNames(hostNamesArg, env); - JNIString appId(appIdArg, env); - - nativeFileDistributionManagerField.get(self, env)->_fileDBModel-> - cleanDeployedFilesToDownload(hostNames._value, appId._value); - } STANDARDCATCH() -} - -JNIEXPORT -void JNICALL Java_com_yahoo_vespa_filedistribution_FileDistributionManager_removeDeploymentsThatHaveDifferentApplicationIdImpl( JNIEnv *env, jobject self, jobjectArray hostNamesArg, jbyteArray appIdArg) { @@ -200,40 +186,3 @@ Java_com_yahoo_vespa_filedistribution_FileDistributionManager_removeDeploymentsT } STANDARDCATCH() } - - -JNIEXPORT -void JNICALL -Java_com_yahoo_vespa_filedistribution_FileDistributionManager_limitFilesTo( - JNIEnv *env, jobject self, jobjectArray fileReferencesArg) -{ - try { - JNIArray<JNIString> fileReferences(fileReferencesArg, env); - - nativeFileDistributionManagerField.get(self, env)->_fileDBModel-> - cleanFiles(fileReferences._value); - } STANDARDCATCH() -} - - -JNIEXPORT -jbyteArray JNICALL -Java_com_yahoo_vespa_filedistribution_FileDistributionManager_getProgressImpl( - JNIEnv *env, jobject self, jbyteArray fileReferenceArg, jobjectArray hostNamesArg) -{ - try { - JNIString fileReference(fileReferenceArg, env); - JNIArray<JNIString> hostNames(hostNamesArg, env); - - const FileDBModel::Progress progress = - nativeFileDistributionManagerField.get(self, env)->_fileDBModel-> - getProgress(fileReference._value, hostNames._value); - - jbyteArray result = env->NewByteArray(progress.size()); - if (!result) - return 0; //exception thrown when returning - - env->SetByteArrayRegion(result, 0, progress.size(), &*progress.begin()); - return result; - } STANDARDCATCH(return 0) -} diff --git a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt index 5b92aa4086d..51af6b106dc 100644 --- a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt @@ -14,7 +14,5 @@ vespa_add_library(filedistribution_filedistributionmodel STATIC zkfiledbmodel.cpp DEPENDS ) -vespa_generate_config(filedistribution_filedistributionmodel filereferences.def) - vespa_add_target_external_dependency(filedistribution_filedistributionmodel zookeeper_mt) -install_config_definition(filereferences.def cloud.config.filedistribution.filereferences.def) + diff --git a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.h b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.h index 2041046c530..5374040a7f1 100644 --- a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.h +++ b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.h @@ -2,7 +2,7 @@ #pragma once #include "filedistributionmodel.h" -#include <vespa/filedistribution/model/config-filereferences.h> +#include <vespa/config-filereferences.h> #include "zkfacade.h" #include "zkfiledbmodel.h" #include <vespa/config/config.h> diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java index 0a803269ff6..f0c463abe70 100644 --- a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java +++ b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.filedistribution; import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.List; import java.util.concurrent.locks.Lock; /** @@ -30,13 +29,8 @@ public class FileDistributionManager { private native void setDeployedFilesImpl(byte[] host, byte[] appId, byte[][] fileReferences); - private native void limitSendingOfDeployedFilesToImpl(byte[][] hostNames, byte[] appId); - private native void limitFilesToImpl(byte[][] fileReferences); private native void removeDeploymentsThatHaveDifferentApplicationIdImpl(byte[][] asByteArrays, byte[] bytes); - private native byte[] getProgressImpl(byte[] fileReference, - byte[][] hostNames); - private byte[][] getAsByteArrays(Collection<String> strings) { byte[][] byteArrays = new byte[strings.size()][]; int i = 0; @@ -121,17 +115,6 @@ public class FileDistributionManager { } } - public void limitSendingOfDeployedFilesTo(Collection<String> hostNames) { - try (LockGuard guard = new LockGuard(lock)) { - limitSendingOfDeployedFilesToImpl(getAsByteArrays(hostNames), appId.getBytes()); - } - } - - public byte[] getProgress(String fileReference, - List<String> hostNamesSortedAscending) { - return getProgressImpl(fileReference.getBytes(), getAsByteArrays(hostNamesSortedAscending)); - } - public native void shutdown(); diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index f2a4ab3a23d..ff6c1ab0b1d 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -45,7 +45,7 @@ public: void SyncPacket::Free() { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _done = true; if (_waiting) { _cond.notify_one(); @@ -560,7 +560,7 @@ FNET_Connection::OpenChannel() uint32_t chid; { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); chid = GetNextID(); AddRef_NoLock(); } @@ -652,7 +652,7 @@ FNET_Connection::PostPacket(FNET_Packet *packet, uint32_t chid) uint32_t FNET_Connection::GetQueueLen() { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); return _queue.GetPacketCnt_NoLock() + _myQueue.GetPacketCnt_NoLock(); } @@ -710,7 +710,7 @@ FNET_Connection::HandleReadEvent() bool FNET_Connection::writePendingAfterConnect() { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); _state = FNET_CONNECTED; // SetState(FNET_CONNECTED) LOG(debug, "Connection(%s): State transition: %s -> %s", GetSpec(), GetStateString(FNET_CONNECTING), GetStateString(FNET_CONNECTED)); diff --git a/fnet/src/vespa/fnet/frt/invoker.cpp b/fnet/src/vespa/fnet/frt/invoker.cpp index 5afc355c68f..ce4daa988de 100644 --- a/fnet/src/vespa/fnet/frt/invoker.cpp +++ b/fnet/src/vespa/fnet/frt/invoker.cpp @@ -32,7 +32,7 @@ void FRT_SingleReqWait::RequestDone(FRT_RPCRequest *req) { (void) req; - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _done = true; if (_waiting) { _cond.notify_one(); diff --git a/fnet/src/vespa/fnet/iocomponent.cpp b/fnet/src/vespa/fnet/iocomponent.cpp index ec51c1f080e..8276da57e2e 100644 --- a/fnet/src/vespa/fnet/iocomponent.cpp +++ b/fnet/src/vespa/fnet/iocomponent.cpp @@ -48,7 +48,7 @@ FNET_IOComponent::UpdateTimeOut() { void FNET_IOComponent::AddRef() { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); assert(_ioc_refcnt > 0); _ioc_refcnt++; } @@ -66,7 +66,7 @@ void FNET_IOComponent::SubRef() { { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); assert(_ioc_refcnt > 0); if (--_ioc_refcnt > 0) { return; diff --git a/fnet/src/vespa/fnet/packetqueue.cpp b/fnet/src/vespa/fnet/packetqueue.cpp index 4fdb5842a9d..4331819d3f5 100644 --- a/fnet/src/vespa/fnet/packetqueue.cpp +++ b/fnet/src/vespa/fnet/packetqueue.cpp @@ -192,7 +192,7 @@ void FNET_PacketQueue::QueuePacket(FNET_Packet *packet, FNET_Context context) { assert(packet != nullptr); - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); EnsureFree(); _buf[_in_pos]._packet = packet; // insert packet ref. _buf[_in_pos]._context = context; @@ -257,7 +257,7 @@ FNET_PacketQueue::DequeuePacket(uint32_t maxwait, FNET_Context *context) void FNET_PacketQueue::Print(uint32_t indent) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); uint32_t i = _out_pos; uint32_t cnt = _bufused; diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp index ef67407cb44..6c8340c1ff8 100644 --- a/fnet/src/vespa/fnet/scheduler.cpp +++ b/fnet/src/vespa/fnet/scheduler.cpp @@ -40,7 +40,7 @@ FNET_Scheduler::~FNET_Scheduler() bool empty = true; std::stringstream dump; { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); dump << "FNET_Scheduler {" << std::endl; dump << " [slot=" << _currSlot << "][iter=" << _currIter << "]" << std::endl; for (int i = 0; i <= NUM_SLOTS; i++) { @@ -70,7 +70,7 @@ FNET_Scheduler::Schedule(FNET_Task *task, double seconds) { uint32_t ticks = 1 + (uint32_t) (seconds * (1000 / SLOT_TICK) + 0.5); - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (!task->_killed) { if (IsActive(task)) LinkOut(task); @@ -84,7 +84,7 @@ FNET_Scheduler::Schedule(FNET_Task *task, double seconds) void FNET_Scheduler::ScheduleNow(FNET_Task *task) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (!task->_killed) { if (IsActive(task)) LinkOut(task); @@ -119,7 +119,7 @@ FNET_Scheduler::Kill(FNET_Task *task) void FNET_Scheduler::Print(FILE *dst) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); fprintf(dst, "FNET_Scheduler {\n"); fprintf(dst, " [slot=%d][iter=%d]\n", _currSlot, _currIter); for (int i = 0; i <= NUM_SLOTS; i++) { diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 61fcc26f1c6..443c90f1af4 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -173,7 +173,7 @@ FNET_TransportThread::UpdateStats() comp->FlushDirectWriteStats(); } { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _stats.Update(&_counters, ms / 1000.0); } _counters.Clear(); @@ -238,7 +238,7 @@ FNET_TransportThread::FNET_TransportThread(FNET_Transport &owner_in) FNET_TransportThread::~FNET_TransportThread() { { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _deleted = true; } if (_started && !_finished) { @@ -379,7 +379,7 @@ FNET_TransportThread::ShutDown(bool waitFinished) { bool wasEmpty = false; { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (!_shutdown) { _shutdown = true; wasEmpty = _queue.IsEmpty_NoLock(); @@ -413,7 +413,7 @@ FNET_TransportThread::InitEventLoop() bool wasStarted; bool wasDeleted; { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); wasStarted = _started; wasDeleted = _deleted; if (!_started && !_deleted) { @@ -440,7 +440,7 @@ void FNET_TransportThread::handle_wakeup() { { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); CountEvent(_queue.FlushPackets_NoLock(&_myQueue)); } @@ -590,7 +590,7 @@ FNET_TransportThread::EventLoopIteration() // flush event queue { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _queue.FlushPackets_NoLock(&_myQueue); } @@ -623,7 +623,7 @@ FNET_TransportThread::EventLoopIteration() _myQueue.IsEmpty_NoLock()); { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _finished = true; if (_waitFinished) { _cond.notify_all(); diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsClientFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsClientFactory.java index 4d5279ebfd4..d004ac3af45 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsClientFactory.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsClientFactory.java @@ -9,5 +9,5 @@ import com.yahoo.vespa.applicationmodel.HostName; * @author bakksjo */ public interface JaxRsClientFactory { - <T> T createClient(Class<T> apiClass, HostName hostName, int port, String pathPrefix); + <T> T createClient(Class<T> apiClass, HostName hostName, int port, String pathPrefix, String scheme); } diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java index 722652557c7..6523a0c138f 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsStrategyFactory.java @@ -34,13 +34,15 @@ import java.util.Set; public class JaxRsStrategyFactory { private final Set<HostName> hostNames; private int port; + private final String scheme; private final JaxRsClientFactory jaxRsClientFactory; // TODO: We might need to support per-host port specification. public JaxRsStrategyFactory( final Set<HostName> hostNames, final int port, - final JaxRsClientFactory jaxRsClientFactory) { + final JaxRsClientFactory jaxRsClientFactory, + String scheme) { if (hostNames.isEmpty()) { throw new IllegalArgumentException("hostNames argument must not be empty"); } @@ -48,19 +50,20 @@ public class JaxRsStrategyFactory { this.hostNames = hostNames; this.port = port; this.jaxRsClientFactory = jaxRsClientFactory; + this.scheme = scheme; } public <T> JaxRsStrategy<T> apiWithRetries(final Class<T> apiClass, final String pathPrefix) { Objects.requireNonNull(apiClass, "apiClass argument may not be null"); Objects.requireNonNull(pathPrefix, "pathPrefix argument may not be null"); - return new RetryingJaxRsStrategy<T>(hostNames, port, jaxRsClientFactory, apiClass, pathPrefix); + return new RetryingJaxRsStrategy<T>(hostNames, port, jaxRsClientFactory, apiClass, pathPrefix, scheme); } public <T> JaxRsStrategy<T> apiNoRetries(final Class<T> apiClass, final String pathPrefix) { Objects.requireNonNull(apiClass, "apiClass argument may not be null"); Objects.requireNonNull(pathPrefix, "pathPrefix argument may not be null"); final HostName hostName = getRandom(hostNames); - return new NoRetryJaxRsStrategy<T>(hostName, port, jaxRsClientFactory, apiClass, pathPrefix); + return new NoRetryJaxRsStrategy<T>(hostName, port, jaxRsClientFactory, apiClass, pathPrefix, scheme); } private static final Random random = new Random(); diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java index 8d096a0dff6..5fd72432702 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java @@ -36,8 +36,8 @@ public class JerseyJaxRsClientFactory implements JaxRsClientFactory { * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/HttpUrlConnectorProvider.html#SET_METHOD_WORKAROUND */ @Override - public <T> T createClient(final Class<T> apiClass, final HostName hostName, final int port, final String pathPrefix) { - final UriBuilder uriBuilder = UriBuilder.fromPath(pathPrefix).host(hostName.s()).port(port).scheme("http"); + public <T> T createClient(final Class<T> apiClass, final HostName hostName, final int port, final String pathPrefix, String scheme) { + final UriBuilder uriBuilder = UriBuilder.fromPath(pathPrefix).host(hostName.s()).port(port).scheme(scheme); final Client webClient = ClientBuilder.newClient() .property(ClientProperties.CONNECT_TIMEOUT, connectTimeoutMs) .property(ClientProperties.READ_TIMEOUT, readTimeoutMs) diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategy.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategy.java index 0f053332d99..7d70c37a40d 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategy.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategy.java @@ -18,6 +18,7 @@ public class NoRetryJaxRsStrategy<T> implements JaxRsStrategy<T> { private final int port; private final JaxRsClientFactory jaxRsClientFactory; private final Class<T> apiClass; + private final String scheme; private String pathPrefix; public NoRetryJaxRsStrategy( @@ -25,7 +26,8 @@ public class NoRetryJaxRsStrategy<T> implements JaxRsStrategy<T> { final int port, final JaxRsClientFactory jaxRsClientFactory, final Class<T> apiClass, - final String pathPrefix) { + final String pathPrefix, + String scheme) { Objects.requireNonNull(hostName, "hostName argument may not be null"); Objects.requireNonNull(jaxRsClientFactory, "jaxRsClientFactory argument may not be null"); Objects.requireNonNull(apiClass, "apiClass argument may not be null"); @@ -35,11 +37,12 @@ public class NoRetryJaxRsStrategy<T> implements JaxRsStrategy<T> { this.jaxRsClientFactory = jaxRsClientFactory; this.apiClass = apiClass; this.pathPrefix = pathPrefix; + this.scheme = scheme; } @Override public <R> R apply(final Function<T, R> function) throws IOException { - final T jaxRsClient = jaxRsClientFactory.createClient(apiClass, hostName, port, pathPrefix); + final T jaxRsClient = jaxRsClientFactory.createClient(apiClass, hostName, port, pathPrefix, scheme); try { return function.apply(jaxRsClient); } catch (ProcessingException e) { diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java index a73297780c6..73320a4c72d 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java @@ -28,13 +28,15 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { private final JaxRsClientFactory jaxRsClientFactory; private final Class<T> apiClass; private String pathPrefix; + private final String scheme; public RetryingJaxRsStrategy( final Set<HostName> hostNames, final int port, final JaxRsClientFactory jaxRsClientFactory, final Class<T> apiClass, - final String pathPrefix) { + final String pathPrefix, + String scheme) { if (hostNames.isEmpty()) { throw new IllegalArgumentException("hostNames argument must not be empty"); } @@ -47,6 +49,7 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { this.jaxRsClientFactory = jaxRsClientFactory; this.apiClass = apiClass; this.pathPrefix = pathPrefix; + this.scheme = scheme; } @Override @@ -55,7 +58,7 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { for (int i = 0; i < NUM_LOOP_ATTEMPTS; ++i) { for (final HostName hostName : hostNames) { - final T jaxRsClient = jaxRsClientFactory.createClient(apiClass, hostName, port, pathPrefix); + final T jaxRsClient = jaxRsClientFactory.createClient(apiClass, hostName, port, pathPrefix, scheme); try { return function.apply(jaxRsClient); } catch (ProcessingException e) { diff --git a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/HttpPatchTest.java b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/HttpPatchTest.java index d50efec261a..63e2b814c24 100644 --- a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/HttpPatchTest.java +++ b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/HttpPatchTest.java @@ -70,7 +70,7 @@ public class HttpPatchTest extends JerseyTest { final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); final JaxRsStrategyFactory factory = new JaxRsStrategyFactory( - Collections.singleton(apiHost), apiPort, jaxRsClientFactory); + Collections.singleton(apiHost), apiPort, jaxRsClientFactory, "http"); final JaxRsStrategy<TestResourceApi> client = factory.apiNoRetries(TestResourceApi.class, apiPath); final String responseBody; diff --git a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategyTest.java b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategyTest.java index 1851ff27270..474df0f3c8d 100644 --- a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategyTest.java +++ b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/NoRetryJaxRsStrategyTest.java @@ -36,11 +36,11 @@ public class NoRetryJaxRsStrategyTest { private final JaxRsClientFactory jaxRsClientFactory = mock(JaxRsClientFactory.class); private final TestJaxRsApi mockApi = mock(TestJaxRsApi.class); private final JaxRsStrategy<TestJaxRsApi> jaxRsStrategy = new NoRetryJaxRsStrategy<>( - SERVER_HOST, REST_PORT, jaxRsClientFactory, TestJaxRsApi.class, API_PATH); + SERVER_HOST, REST_PORT, jaxRsClientFactory, TestJaxRsApi.class, API_PATH, "http"); @Before public void setup() { - when(jaxRsClientFactory.createClient(eq(TestJaxRsApi.class), any(HostName.class), anyInt(), anyString())) + when(jaxRsClientFactory.createClient(eq(TestJaxRsApi.class), any(HostName.class), anyInt(), anyString(), anyString())) .thenReturn(mockApi); } @@ -51,7 +51,7 @@ public class NoRetryJaxRsStrategyTest { verify(mockApi, times(1)).doSomething(); verify(jaxRsClientFactory, times(1)) - .createClient(eq(TestJaxRsApi.class), eq(SERVER_HOST), eq(REST_PORT), eq(API_PATH)); + .createClient(eq(TestJaxRsApi.class), eq(SERVER_HOST), eq(REST_PORT), eq(API_PATH), eq("http")); } @Test diff --git a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java index d3fad2aa8eb..10dde1ff820 100644 --- a/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java +++ b/jaxrs_client_utils/src/test/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategyTest.java @@ -48,11 +48,11 @@ public class RetryingJaxRsStrategyTest { private final JaxRsClientFactory jaxRsClientFactory = mock(JaxRsClientFactory.class); private final TestJaxRsApi mockApi = mock(TestJaxRsApi.class); private final JaxRsStrategy<TestJaxRsApi> jaxRsStrategy = new RetryingJaxRsStrategy<>( - SERVER_HOSTS, REST_PORT, jaxRsClientFactory, TestJaxRsApi.class, API_PATH); + SERVER_HOSTS, REST_PORT, jaxRsClientFactory, TestJaxRsApi.class, API_PATH, "http"); @Before public void setup() { - when(jaxRsClientFactory.createClient(eq(TestJaxRsApi.class), any(HostName.class), anyInt(), anyString())) + when(jaxRsClientFactory.createClient(eq(TestJaxRsApi.class), any(HostName.class), anyInt(), anyString(), anyString())) .thenReturn(mockApi); } @@ -65,7 +65,7 @@ public class RetryingJaxRsStrategyTest { // Check that one of the supplied hosts is contacted. final ArgumentCaptor<HostName> hostNameCaptor = ArgumentCaptor.forClass(HostName.class); verify(jaxRsClientFactory, times(1)) - .createClient(eq(TestJaxRsApi.class), hostNameCaptor.capture(), eq(REST_PORT), eq(API_PATH)); + .createClient(eq(TestJaxRsApi.class), hostNameCaptor.capture(), eq(REST_PORT), eq(API_PATH), eq("http")); assertThat(SERVER_HOSTS.contains(hostNameCaptor.getValue()), is(true)); } @@ -135,7 +135,7 @@ public class RetryingJaxRsStrategyTest { final JaxRsClientFactory jaxRsClientFactory) { final ArgumentCaptor<HostName> hostNameCaptor = ArgumentCaptor.forClass(HostName.class); verify(jaxRsClientFactory, atLeast(SERVER_HOSTS.size())) - .createClient(eq(TestJaxRsApi.class), hostNameCaptor.capture(), eq(REST_PORT), eq(API_PATH)); + .createClient(eq(TestJaxRsApi.class), hostNameCaptor.capture(), eq(REST_PORT), eq(API_PATH), eq("http")); final Set<HostName> actualServerHostsContacted = new HashSet<>(hostNameCaptor.getAllValues()); assertThat(actualServerHostsContacted, equalTo(SERVER_HOSTS)); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index 489a4c3dc10..7a76c588fb4 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -287,7 +287,7 @@ public class Request extends AbstractResource { } /** - * <p>Returns the allocated number of milliseconds that this Request is allowed to exist. If no timeout has been set + * <p>Returns the allocated number of time units that this Request is allowed to exist. If no timeout has been set * for this Request, this method returns <em>null</em>.</p> * * @param unit The unit to return the timeout in. @@ -306,7 +306,7 @@ public class Request extends AbstractResource { * <em>null</em>.</p> * * @param unit The unit to return the time in. - * @return The number of milliseconds left until this Request times out, or <em>null</em>. + * @return The number of time units left until this Request times out, or <em>null</em>. */ public Long timeRemaining(TimeUnit unit) { if (timeout == null) { @@ -316,6 +316,16 @@ public class Request extends AbstractResource { } /** + * <p>Returns the time that this Request has existed so far. + * + * @param unit The unit to return the time in. + * @return The number of time units elapsed since this Request was created. + */ + public long timeElapsed(TimeUnit unit) { + return unit.convert(container().currentTimeMillis() - creationTime, TimeUnit.MILLISECONDS); + } + + /** * <p>Returns the time at which this Request was created. This is whatever value was returned by {@link * Timer#currentTimeMillis()} when constructing this.</p> * diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java index 5d4974f2dc4..7318b1b38ae 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java @@ -15,6 +15,7 @@ public class BindingMatch<T> { private final UriPattern.Match match; private final T target; + private final UriPattern matched; /** * <p>Constructs a new instance of this class.</p> @@ -22,12 +23,27 @@ public class BindingMatch<T> { * @param match The match information for this instance. * @param target The target of this match. * @throws NullPointerException If any argument is null. + * @deprecated use BindingMatch(UriPattern.Match match, T target, UriPattern matched) */ + @Deprecated public BindingMatch(UriPattern.Match match, T target) { + this(match, target, null); + } + + /** + * <p>Constructs a new instance of this class.</p> + * + * @param match The match information for this instance. + * @param target The target of this match. + * @param matched The matched URI pattern + * @throws NullPointerException If any argument is null. + */ + public BindingMatch(UriPattern.Match match, T target, UriPattern matched) { Objects.requireNonNull(match, "match"); Objects.requireNonNull(target, "target"); this.match = match; this.target = target; + this.matched = matched; } /** @@ -61,4 +77,14 @@ public class BindingMatch<T> { public T target() { return target; } + + /** + * <p>Returns the URI pattern that was matched.</p> + * + * @return The matched pattern. + */ + public UriPattern matched() { + return matched; + } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSet.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSet.java index 7a21e204dd3..1e25846f63c 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSet.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSet.java @@ -38,9 +38,10 @@ public class BindingSet<T> implements Iterable<Map.Entry<UriPattern, T>> { */ public BindingMatch<T> match(URI uri) { for (Map.Entry<UriPattern, T> entry : bindings) { - UriPattern.Match match = entry.getKey().match(uri); + UriPattern pattern = entry.getKey(); + UriPattern.Match match = pattern.match(uri); if (match != null) { - return new BindingMatch<>(match, entry.getValue()); + return new BindingMatch<>(match, entry.getValue(), pattern); } } return null; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerThread.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerThread.java index 4506a63ac6d..1e947d44fcd 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerThread.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerThread.java @@ -22,8 +22,8 @@ public class ContainerThread extends Thread { * Allocates a new ContainerThread object. This constructor calls the parent {@link Thread#Thread(Runnable)} * constructor. * - * @param target The object whose <code>run</code> method is called. - * @param consumer The MetricConsumer of this thread. + * @param target the object whose <code>run</code> method is called. + * @param consumer the MetricConsumer of this thread. */ public ContainerThread(Runnable target, MetricConsumer consumer) { super(target); @@ -56,6 +56,7 @@ public class ContainerThread extends Thread { public Thread newThread(Runnable target) { return new ContainerThread(target, provider.get()); } + } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index fc3997657dc..cd52a724275 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -48,6 +48,8 @@ public class StandaloneMain { System.out.println("debug\tStopped ok."); System.exit(0); } catch (Exception e) { + System.out.print("debug\tUnexpected: "); + e.printStackTrace(); log.log(Level.SEVERE, "Unexpected: ", e); System.exit(6); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java index d2b58090665..e7039e85e5e 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java @@ -24,7 +24,6 @@ import java.io.StringWriter; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.net.URI; import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Set; @@ -35,14 +34,18 @@ import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; import java.util.stream.Stream; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ @SuppressWarnings("UnusedDeclaration") @Beta public abstract class ServerProviderConformanceTest { + + private static final Logger log = Logger.getLogger(ServerProviderConformanceTest.class.getName()); + private static final int NUM_RUNS_EACH_TEST = 10; /** @@ -2790,7 +2793,7 @@ public abstract class ServerProviderConformanceTest { serverProvider.release(); for (int i = 0; i < NUM_RUNS_EACH_TEST; ++i) { - System.out.println("Test run #" + i); + log.fine("Test run #" + i); requestHandler.reset(adapter.newResponseContent()); final U client = adapter.newClient(serverProvider); final boolean withRequestContent = requestType == RequestType.WITH_CONTENT; diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java index e00bdae153d..cd5e07f1224 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java @@ -297,9 +297,10 @@ public class RequestTestCase { public RequestHandler resolveHandler(Request request) { this.asServer = request.isServerRequest(); RequestHandler requestHandler = new MyRequestHandler(); - request.setBindingMatch(new BindingMatch<>( - new UriPattern("http://*/*").match(request.getUri()), - requestHandler)); + UriPattern pattern = new UriPattern("http://*/*"); + request.setBindingMatch(new BindingMatch<>(pattern.match(request.getUri()), + requestHandler, + pattern)); return requestHandler; } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingMatchTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingMatchTestCase.java index 582a1c6685e..21a3ae08c49 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingMatchTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingMatchTestCase.java @@ -18,32 +18,36 @@ public class BindingMatchTestCase { @Test public void requireThatAccessorsWork() { Object obj = new Object(); + UriPattern pattern = new UriPattern("http://*/*"); BindingMatch<Object> match = new BindingMatch<>( - new UriPattern("http://*/*").match(URI.create("http://localhost:69/status.html")), - obj); + pattern.match(URI.create("http://localhost:69/status.html")), + obj, pattern); assertSame(obj, match.target()); assertEquals(3, match.groupCount()); assertEquals("localhost", match.group(0)); assertEquals("69", match.group(1)); assertEquals("status.html", match.group(2)); + assertEquals(pattern, match.matched()); } @Test public void requireThatConstructorArgumentsCanNotBeNull() { try { - new BindingMatch<>(null, null); + new BindingMatch<>(null, null, null); fail(); } catch (NullPointerException e) { } try { - new BindingMatch<>(new UriPattern("http://*/*").match(URI.create("http://localhost/")), null); + UriPattern pattern = new UriPattern("http://*/*"); + new BindingMatch<>(pattern.match(URI.create("http://localhost/")), null, pattern); fail(); } catch (NullPointerException e) { } try { - new BindingMatch<>(null, new Object()); + UriPattern pattern = new UriPattern("http://*/*"); + new BindingMatch<>(null, new Object(), pattern); fail(); } catch (NullPointerException e) { diff --git a/jdisc_core_test/test_bundles/app-h-log/src/main/java/com/yahoo/jdisc/bundle/ApplicationH.java b/jdisc_core_test/test_bundles/app-h-log/src/main/java/com/yahoo/jdisc/bundle/ApplicationH.java index ae37e4e6cfc..f014a16ddb3 100644 --- a/jdisc_core_test/test_bundles/app-h-log/src/main/java/com/yahoo/jdisc/bundle/ApplicationH.java +++ b/jdisc_core_test/test_bundles/app-h-log/src/main/java/com/yahoo/jdisc/bundle/ApplicationH.java @@ -8,7 +8,7 @@ import com.yahoo.jdisc.application.ContainerActivator; import com.yahoo.jdisc.service.CurrentContainer; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class ApplicationH extends AbstractApplication { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java index 2268b568b18..21e492fe57e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java @@ -16,6 +16,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; +import java.security.Principal; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -71,6 +72,7 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { private final HeaderFields trailers = new HeaderFields(); private final Map<String, List<String>> parameters = new HashMap<>(); + private Principal principal; private final long connectedAt; private Method method; private Version version; @@ -294,6 +296,14 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { return version == Version.HTTP_1_1; } + public Principal getUserPrincipal() { + return principal; + } + + public void setUserPrincipal(Principal principal) { + this.principal = principal; + } + public static HttpRequest newServerRequest(CurrentContainer container, URI uri) { return newServerRequest(container, uri, Method.GET); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java index a46d35f8e70..617f0cbd184 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java @@ -36,14 +36,12 @@ public abstract class DiscFilterRequest { protected static final String HTTPS_PREFIX = "https"; protected static final int DEFAULT_HTTP_PORT = 80; protected static final int DEFAULT_HTTPS_PORT = 443; - private static final String JDISC_REQUEST_PRINCIPAL = "jdisc.request.principal"; private final ServletOrJdiscHttpRequest parent; protected final InetSocketAddress localAddress; protected final Map<String, List<String>> untreatedParams; private final HeaderFields untreatedHeaders; private List<Cookie> untreatedCookies = null; - private Principal userPrincipal = null; private String remoteUser = null; private String[] roles = null; private boolean overrideIsUserInRole = false; @@ -330,9 +328,7 @@ public abstract class DiscFilterRequest { return port; } - public Principal getUserPrincipal() { - return (Principal) getAttribute(JDISC_REQUEST_PRINCIPAL); - } + public abstract Principal getUserPrincipal(); public boolean isSecure() { if(getScheme().equalsIgnoreCase(HTTPS_PREFIX)) { @@ -375,9 +371,7 @@ public abstract class DiscFilterRequest { this.remoteUser = remoteUser; } - public void setUserPrincipal(Principal principal) { - setAttribute(JDISC_REQUEST_PRINCIPAL, principal); - } + public abstract void setUserPrincipal(Principal principal); public void setUserRoles(String[] roles) { this.roles = roles; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java index 1e9d09ecb17..07e3b97ba90 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java @@ -5,6 +5,7 @@ import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; import java.net.URI; +import java.security.Principal; import java.util.ArrayList; import java.util.Collections; import java.util.Enumeration; @@ -103,6 +104,16 @@ public class JdiscFilterRequest extends DiscFilterRequest { } @Override + public Principal getUserPrincipal() { + return parent.getUserPrincipal(); + } + + @Override + public void setUserPrincipal(Principal principal) { + this.parent.setUserPrincipal(principal); + } + + @Override public void clearCookies() { parent.headers().remove(HttpHeaders.Names.COOKIE); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java index 0fd52d3f12a..11c2baf0176 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java @@ -6,6 +6,7 @@ import com.yahoo.jdisc.http.servlet.ServletRequest; import java.io.UnsupportedEncodingException; import java.net.URI; +import java.security.Principal; import java.util.Collections; import java.util.Enumeration; import java.util.HashSet; @@ -128,6 +129,16 @@ class ServletFilterRequest extends DiscFilterRequest { } @Override + public Principal getUserPrincipal() { + return parent.getUserPrincipal(); + } + + @Override + public void setUserPrincipal(Principal principal) { + parent.setUserPrincipal(principal); + } + + @Override public void removeHeaders(String name) { parent.removeHeaders(name); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 3363c7d3284..c3c83474e56 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -5,6 +5,7 @@ import com.google.common.base.Objects; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.http.servlet.ServletRequest; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.RequestLog; import org.eclipse.jetty.server.Response; @@ -17,6 +18,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.security.Principal; import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -107,6 +109,12 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog accessLogEntry.setPeerPort(peerPort); } accessLogEntry.setHttpVersion(request.getProtocol()); + accessLogEntry.setScheme(request.getScheme()); + accessLogEntry.setLocalPort(request.getLocalPort()); + Principal principal = (Principal) request.getAttribute(ServletRequest.JDISC_REQUEST_PRINCIPAL); + if (principal != null) { + accessLogEntry.setUserPrincipal(principal); + } } private static String getRemoteAddress(final HttpServletRequest request) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java index 43513b4efba..e30d50ecdbf 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLoggingRequestHandler.java @@ -54,24 +54,27 @@ public class AccessLoggingRequestHandler extends AbstractRequestHandler { Preconditions.checkArgument(request instanceof HttpRequest, "Expected HttpRequest, got " + request); final HttpRequest httpRequest = (HttpRequest) request; httpRequest.context().put(CONTEXT_KEY_ACCESS_LOG_ENTRY, accessLogEntry); - final ResponseHandler accessLoggingResponseHandler = new AccessLoggingResponseHandler(handler, accessLogEntry); + final ResponseHandler accessLoggingResponseHandler = new AccessLoggingResponseHandler(httpRequest, handler, accessLogEntry); final ContentChannel requestContentChannel = delegate.handleRequest(request, accessLoggingResponseHandler); return requestContentChannel; } private static class AccessLoggingResponseHandler implements ResponseHandler { + private final HttpRequest request; private final ResponseHandler delegateHandler; private final AccessLogEntry accessLogEntry; public AccessLoggingResponseHandler( - final ResponseHandler delegateHandler, + HttpRequest request, final ResponseHandler delegateHandler, final AccessLogEntry accessLogEntry) { + this.request = request; this.delegateHandler = delegateHandler; this.accessLogEntry = accessLogEntry; } @Override public ContentChannel handleResponse(Response response) { + accessLogEntry.setUserPrincipal(request.getUserPrincipal()); return delegateHandler.handleResponse(response); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index 96180f48229..8255e16e0ee 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -1,66 +1,41 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.google.common.base.Preconditions; import com.google.inject.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Ssl; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl.PemKeyStore; import com.yahoo.jdisc.http.SecretStore; -import com.yahoo.jdisc.http.ssl.ReaderForPath; -import com.yahoo.jdisc.http.ssl.SslKeyStore; -import com.yahoo.jdisc.http.ssl.pem.PemSslKeyStore; +import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreContext; +import com.yahoo.jdisc.http.ssl.SslKeyStoreConfigurator; import org.eclipse.jetty.http.HttpVersion; -import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnectionStatistics; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; -import javax.servlet.ServletRequest; -import java.io.IOException; -import java.io.Reader; -import java.lang.reflect.Field; -import java.net.Socket; -import java.net.SocketException; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.nio.channels.ServerSocketChannel; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyStore; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static com.google.common.io.Closeables.closeQuietly; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType.Enum.JKS; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType.Enum.PEM; -import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked; /** * @author Einar M R Rosenvinge + * @author bjorncs */ public class ConnectorFactory { - private final static Logger log = Logger.getLogger(ConnectorFactory.class.getName()); private final ConnectorConfig connectorConfig; private final SecretStore secretStore; + private final SslKeyStoreConfigurator sslKeyStoreConfigurator; @Inject - public ConnectorFactory(ConnectorConfig connectorConfig, SecretStore secretStore) { + public ConnectorFactory(ConnectorConfig connectorConfig, + SecretStore secretStore, + SslKeyStoreConfigurator sslKeyStoreConfigurator) { this.connectorConfig = connectorConfig; this.secretStore = secretStore; + this.sslKeyStoreConfigurator = sslKeyStoreConfigurator; if (connectorConfig.ssl().enabled()) validateSslConfig(connectorConfig); @@ -69,14 +44,8 @@ public class ConnectorFactory { // TODO: can be removed when we have dedicated SSL config in services.xml private static void validateSslConfig(ConnectorConfig config) { ConnectorConfig.Ssl ssl = config.ssl(); - - if (ssl.keyStoreType() == JKS) { - if (! ssl.pemKeyStore().keyPath().isEmpty() || ! ssl.pemKeyStore().certificatePath().isEmpty()) - throw new IllegalArgumentException("pemKeyStore attributes can not be set when keyStoreType is JKS."); - } - if (ssl.keyStoreType() == PEM) { - if (! ssl.keyStorePath().isEmpty()) - throw new IllegalArgumentException("keyStorePath can not be set when keyStoreType is PEM"); + if (!ssl.trustStorePath().isEmpty() && ssl.useTrustStorePassword() && ssl.keyDbKey().isEmpty()) { + throw new IllegalArgumentException("Missing password for JKS truststore"); } } @@ -84,11 +53,11 @@ public class ConnectorFactory { return connectorConfig; } - public ServerConnector createConnector(final Metric metric, final Server server, final ServerSocketChannel ch, Map<Path, FileChannel> keyStoreChannels) { + public ServerConnector createConnector(final Metric metric, final Server server, final ServerSocketChannel ch) { ServerConnector connector; if (connectorConfig.ssl().enabled()) { connector = new JDiscServerConnector(connectorConfig, metric, server, ch, - newSslConnectionFactory(keyStoreChannels), + newSslConnectionFactory(), newHttpConnectionFactory()); } else { connector = new JDiscServerConnector(connectorConfig, metric, server, ch, @@ -125,10 +94,13 @@ public class ConnectorFactory { } //TODO: does not support loading non-yahoo readable JKS key stores. - private SslConnectionFactory newSslConnectionFactory(Map<Path, FileChannel> keyStoreChannels) { + private SslConnectionFactory newSslConnectionFactory() { Ssl sslConfig = connectorConfig.ssl(); SslContextFactory factory = new SslContextFactory(); + + sslKeyStoreConfigurator.configure(new DefaultSslKeyStoreContext(factory)); + switch (sslConfig.clientAuth()) { case NEED_AUTH: factory.setNeedClientAuth(true); @@ -172,25 +144,14 @@ public class ConnectorFactory { factory.setIncludeCipherSuites(ciphs); } - Optional<String> keyDbPassword = secret(sslConfig.keyDbKey()); - switch (sslConfig.keyStoreType()) { - case PEM: - factory.setKeyStore(getKeyStore(sslConfig.pemKeyStore(), keyStoreChannels)); - if (keyDbPassword.isPresent()) - log.warning("Encrypted PEM key stores are not supported."); - break; - case JKS: - factory.setKeyStorePath(sslConfig.keyStorePath()); - factory.setKeyStoreType(sslConfig.keyStoreType().toString()); - factory.setKeyStorePassword(keyDbPassword.orElseThrow(passwordRequiredForJKSKeyStore("key"))); - break; - } + String keyDbPassword = sslConfig.keyDbKey(); if (!sslConfig.trustStorePath().isEmpty()) { factory.setTrustStorePath(sslConfig.trustStorePath()); factory.setTrustStoreType(sslConfig.trustStoreType().toString()); - if (sslConfig.useTrustStorePassword()) - factory.setTrustStorePassword(keyDbPassword.orElseThrow(passwordRequiredForJKSKeyStore("trust"))); + if (sslConfig.useTrustStorePassword()) { + factory.setTrustStorePassword(secretStore.getSecret(keyDbPassword)); + } } factory.setKeyManagerFactoryAlgorithm(sslConfig.sslKeyManagerFactoryAlgorithm()); @@ -198,162 +159,4 @@ public class ConnectorFactory { return new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()); } - /** Returns the secret password with the given name, or empty if the password name is null or empty */ - private Optional<String> secret(String keyname) { - return Optional.of(keyname).filter(key -> !key.isEmpty()).map(secretStore::getSecret); - } - - @SuppressWarnings("ThrowableInstanceNeverThrown") - private Supplier<RuntimeException> passwordRequiredForJKSKeyStore(String type) { - return () -> new RuntimeException(String.format("Password is required for JKS %s store", type)); - } - - private KeyStore getKeyStore(PemKeyStore pemKeyStore, Map<Path, FileChannel> keyStoreChannels) { - Preconditions.checkArgument(!pemKeyStore.certificatePath().isEmpty(), "Missing certificate path."); - Preconditions.checkArgument(!pemKeyStore.keyPath().isEmpty(), "Missing key path."); - - class KeyStoreReaderForPath implements AutoCloseable { - private final Optional<FileChannel> channel; - public final ReaderForPath readerForPath; - - - KeyStoreReaderForPath(String pathString) { - Path path = Paths.get(pathString); - channel = Optional.ofNullable(keyStoreChannels.get(path)); - readerForPath = new ReaderForPath(channel.map(this::getReader).orElseGet(() -> getReader(path)), path); - } - - private Reader getReader(FileChannel channel) { - try { - channel.position(0); - return Channels.newReader(channel, StandardCharsets.UTF_8.newDecoder(), -1); - } catch (IOException e) { - throw throwUnchecked(e); - } - - } - - private Reader getReader(Path path) { - try { - return Files.newBufferedReader(path); - } catch (IOException e) { - throw new RuntimeException("Failed opening " + path, e); - } - } - - @Override - public void close() { - //channels are reused - if (!channel.isPresent()) { - closeQuietly(readerForPath.reader); - } - } - } - - try (KeyStoreReaderForPath certificateReader = new KeyStoreReaderForPath(pemKeyStore.certificatePath()); - KeyStoreReaderForPath keyReader = new KeyStoreReaderForPath(pemKeyStore.keyPath())) { - SslKeyStore keyStore = new PemSslKeyStore( - new com.yahoo.jdisc.http.ssl.pem.PemKeyStore.KeyStoreLoadParameter( - certificateReader.readerForPath, keyReader.readerForPath)); - return keyStore.loadJavaKeyStore(); - } catch (Exception e) { - throw new RuntimeException("Failed setting up key store for " + pemKeyStore.keyPath() + ", " + pemKeyStore.certificatePath(), e); - } - } - - public static class JDiscServerConnector extends ServerConnector { - public static final String REQUEST_ATTRIBUTE = JDiscServerConnector.class.getName(); - private final static Logger log = Logger.getLogger(JDiscServerConnector.class.getName()); - private final Metric.Context metricCtx; - private final ServerConnectionStatistics statistics; - private final boolean tcpKeepAlive; - private final boolean tcpNoDelay; - private final ServerSocketChannel channelOpenedByActivator; - - private JDiscServerConnector(ConnectorConfig config, Metric metric, Server server, - ServerSocketChannel channelOpenedByActivator, ConnectionFactory... factories) { - super(server, factories); - this.channelOpenedByActivator = channelOpenedByActivator; - this.tcpKeepAlive = config.tcpKeepAliveEnabled(); - this.tcpNoDelay = config.tcpNoDelay(); - this.metricCtx = createMetricContext(config, metric); - - this.statistics = new ServerConnectionStatistics(); - addBean(statistics); - } - - private Metric.Context createMetricContext(ConnectorConfig config, Metric metric) { - Map<String, Object> props = new TreeMap<>(); - props.put(JettyHttpServer.Metrics.NAME_DIMENSION, config.name()); - props.put(JettyHttpServer.Metrics.PORT_DIMENSION, config.listenPort()); - return metric.createContext(props); - } - - @Override - protected void configure(final Socket socket) { - super.configure(socket); - try { - socket.setKeepAlive(tcpKeepAlive); - socket.setTcpNoDelay(tcpNoDelay); - } catch (SocketException ignored) { - } - } - - @Override - public void open() throws IOException { - if (channelOpenedByActivator == null) { - log.log(Level.INFO, "No channel set by activator, opening channel ourselves."); - try { - super.open(); - } catch (RuntimeException e) { - log.log(Level.SEVERE, "failed org.eclipse.jetty.server.Server open() with port "+getPort()); - throw e; - } - return; - } - log.log(Level.INFO, "Using channel set by activator: " + channelOpenedByActivator); - - channelOpenedByActivator.socket().setReuseAddress(getReuseAddress()); - int localPort = channelOpenedByActivator.socket().getLocalPort(); - try { - uglySetLocalPort(localPort); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException("Could not set local port.", e); - } - if (localPort <= 0) { - throw new IOException("Server channel not bound"); - } - addBean(channelOpenedByActivator); - channelOpenedByActivator.configureBlocking(true); - addBean(channelOpenedByActivator); - - try { - uglySetChannel(channelOpenedByActivator); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException("Could not set server channel.", e); - } - } - - private void uglySetLocalPort(int localPort) throws NoSuchFieldException, IllegalAccessException { - Field localPortField = ServerConnector.class.getDeclaredField("_localPort"); - localPortField.setAccessible(true); - localPortField.set(this, localPort); - } - - private void uglySetChannel(ServerSocketChannel channelOpenedByActivator) throws NoSuchFieldException, - IllegalAccessException { - Field acceptChannelField = ServerConnector.class.getDeclaredField("_acceptChannel"); - acceptChannelField.setAccessible(true); - acceptChannelField.set(this, channelOpenedByActivator); - } - - public ServerConnectionStatistics getStatistics() { return statistics; } - - public Metric.Context getMetricContext() { return metricCtx; } - - public static JDiscServerConnector fromRequest(ServletRequest request) { - return (JDiscServerConnector)request.getAttribute(REQUEST_ATTRIBUTE); - } - } - } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 7f169c7c8d0..5cabe8acd27 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -12,6 +12,7 @@ import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.jdisc.http.HttpRequest; +import org.eclipse.jetty.io.EofException; import org.eclipse.jetty.server.HttpConnection; import javax.servlet.AsyncContext; @@ -122,7 +123,11 @@ class HttpRequestDispatch { boolean reportedError = false; if (error != null) { - if (!(error instanceof OverloadException || error instanceof BindingNotFoundException)) { + if (error instanceof EofException) { + log.log(Level.FINE, + "Network connection was unexpectedly terminated: " + parent.servletRequest.getRequestURI(), + error); + } else if (!(error instanceof OverloadException || error instanceof BindingNotFoundException)) { log.log(Level.WARNING, "Request failed: " + parent.servletRequest.getRequestURI(), error); } reportedError = true; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index 543cf8ab43e..27f72c7b4bf 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -20,7 +20,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection; -import static com.yahoo.jdisc.http.server.jetty.ConnectorFactory.JDiscServerConnector; /** * @author Simon Thoresen Hult diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java new file mode 100644 index 00000000000..8dd50074c32 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java @@ -0,0 +1,122 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.http.ConnectorConfig; +import org.eclipse.jetty.server.ConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnectionStatistics; +import org.eclipse.jetty.server.ServerConnector; + +import javax.servlet.ServletRequest; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.Socket; +import java.net.SocketException; +import java.nio.channels.ServerSocketChannel; +import java.util.Map; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +class JDiscServerConnector extends ServerConnector { + public static final String REQUEST_ATTRIBUTE = JDiscServerConnector.class.getName(); + private final static Logger log = Logger.getLogger(JDiscServerConnector.class.getName()); + private final Metric.Context metricCtx; + private final ServerConnectionStatistics statistics; + private final boolean tcpKeepAlive; + private final boolean tcpNoDelay; + private final ServerSocketChannel channelOpenedByActivator; + + JDiscServerConnector(ConnectorConfig config, Metric metric, Server server, + ServerSocketChannel channelOpenedByActivator, ConnectionFactory... factories) { + super(server, factories); + this.channelOpenedByActivator = channelOpenedByActivator; + this.tcpKeepAlive = config.tcpKeepAliveEnabled(); + this.tcpNoDelay = config.tcpNoDelay(); + this.metricCtx = createMetricContext(config, metric); + + this.statistics = new ServerConnectionStatistics(); + addBean(statistics); + } + + private Metric.Context createMetricContext(ConnectorConfig config, Metric metric) { + Map<String, Object> props = new TreeMap<>(); + props.put(JettyHttpServer.Metrics.NAME_DIMENSION, config.name()); + props.put(JettyHttpServer.Metrics.PORT_DIMENSION, config.listenPort()); + return metric.createContext(props); + } + + @Override + protected void configure(final Socket socket) { + super.configure(socket); + try { + socket.setKeepAlive(tcpKeepAlive); + socket.setTcpNoDelay(tcpNoDelay); + } catch (SocketException ignored) { + } + } + + @Override + public void open() throws IOException { + if (channelOpenedByActivator == null) { + log.log(Level.INFO, "No channel set by activator, opening channel ourselves."); + try { + super.open(); + } catch (RuntimeException e) { + log.log(Level.SEVERE, "failed org.eclipse.jetty.server.Server open() with port " + getPort()); + throw e; + } + return; + } + log.log(Level.INFO, "Using channel set by activator: " + channelOpenedByActivator); + + channelOpenedByActivator.socket().setReuseAddress(getReuseAddress()); + int localPort = channelOpenedByActivator.socket().getLocalPort(); + try { + uglySetLocalPort(localPort); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException("Could not set local port.", e); + } + if (localPort <= 0) { + throw new IOException("Server channel not bound"); + } + addBean(channelOpenedByActivator); + channelOpenedByActivator.configureBlocking(true); + addBean(channelOpenedByActivator); + + try { + uglySetChannel(channelOpenedByActivator); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException("Could not set server channel.", e); + } + } + + private void uglySetLocalPort(int localPort) throws NoSuchFieldException, IllegalAccessException { + Field localPortField = ServerConnector.class.getDeclaredField("_localPort"); + localPortField.setAccessible(true); + localPortField.set(this, localPort); + } + + private void uglySetChannel(ServerSocketChannel channelOpenedByActivator) throws NoSuchFieldException, + IllegalAccessException { + Field acceptChannelField = ServerConnector.class.getDeclaredField("_acceptChannel"); + acceptChannelField.setAccessible(true); + acceptChannelField.set(this, channelOpenedByActivator); + } + + public ServerConnectionStatistics getStatistics() { + return statistics; + } + + public Metric.Context getMetricContext() { + return metricCtx; + } + + public static JDiscServerConnector fromRequest(ServletRequest request) { + return (JDiscServerConnector) request.getAttribute(REQUEST_ATTRIBUTE); + } +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 509bf42d466..7bff685e780 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -44,15 +44,11 @@ import javax.servlet.DispatcherType; import java.lang.management.ManagementFactory; import java.net.BindException; import java.net.MalformedURLException; -import java.nio.channels.FileChannel; import java.nio.channels.ServerSocketChannel; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.EnumSet; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -62,9 +58,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.jdisc.http.server.jetty.ConnectorFactory.JDiscServerConnector; -import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked; - /** * @author Simon Thoresen Hult * @author bjorncs @@ -147,11 +140,9 @@ public class JettyHttpServer extends AbstractServerProvider { setupJmx(server, serverConfig); ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(serverConfig.maxWorkerThreads()); - Map<Path, FileChannel> keyStoreChannels = getKeyStoreFileChannels(osgiFramework.bundleContext()); - for (ConnectorFactory connectorFactory : connectorFactories.allComponents()) { ServerSocketChannel preBoundChannel = getChannelFromServiceLayer(connectorFactory.getConnectorConfig().listenPort(), osgiFramework.bundleContext()); - server.addConnector(connectorFactory.createConnector(metric, server, preBoundChannel, keyStoreChannels)); + server.addConnector(connectorFactory.createConnector(metric, server, preBoundChannel)); listenedPorts.add(connectorFactory.getConnectorConfig().listenPort()); } @@ -257,43 +248,6 @@ public class JettyHttpServer extends AbstractServerProvider { return "/" + servletPathsConfig.servlets(id.stringValue()).path(); } - // Ugly trick to get generic type literal. - @SuppressWarnings("unchecked") - private static final Class<Map<?, ?>> mapClass = (Class<Map<?, ?>>) (Object) Map.class; - - private Map<Path, FileChannel> getKeyStoreFileChannels(BundleContext bundleContext) { - try { - Collection<ServiceReference<Map<?, ?>>> serviceReferences = bundleContext.getServiceReferences(mapClass, - "(role=com.yahoo.container.standalone.StandaloneContainerActivator.KeyStoreFileChannels)"); - - if (serviceReferences == null || serviceReferences.isEmpty()) - return Collections.emptyMap(); - - if (serviceReferences.size() != 1) - throw new IllegalStateException("Multiple KeyStoreFileChannels registered"); - - return getKeyStoreFileChannels(bundleContext, serviceReferences.iterator().next()); - } catch (InvalidSyntaxException e) { - throw throwUnchecked(e); - } - } - - @SuppressWarnings("unchecked") - private Map<Path, FileChannel> getKeyStoreFileChannels(BundleContext bundleContext, ServiceReference<Map<?, ?>> keyStoreFileChannelReference) { - Map<?, ?> fileChannelMap = bundleContext.getService(keyStoreFileChannelReference); - try { - if (fileChannelMap == null) - return Collections.emptyMap(); - - Map<Path, FileChannel> result = (Map<Path, FileChannel>) fileChannelMap; - log.fine("Using file channel for " + result.keySet()); - return result; - } finally { - //if we change this to be anything other than a simple map, we should hold the reference as long as the object is in use. - bundleContext.ungetService(keyStoreFileChannelReference); - } - } - private ServletContextHandler createServletContextHandler() { ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.NO_SECURITY | ServletContextHandler.NO_SESSIONS); servletContextHandler.setContextPath("/"); @@ -324,8 +278,8 @@ public class JettyHttpServer extends AbstractServerProvider { return bundleContext.getService(ref); } - private static ExecutorService newJanitor(final ThreadFactory factory) { - final int threadPoolSize = Runtime.getRuntime().availableProcessors(); + private static ExecutorService newJanitor(ThreadFactory factory) { + int threadPoolSize = Runtime.getRuntime().availableProcessors(); log.info("Creating janitor executor with " + threadPoolSize + " threads"); return Executors.newFixedThreadPool( threadPoolSize, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java index 3cbe415d39d..db8780b087c 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java @@ -12,6 +12,7 @@ import javax.servlet.http.HttpServletRequestWrapper; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; +import java.security.Principal; import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; @@ -36,6 +37,7 @@ import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection; * @since 5.27 */ public class ServletRequest extends HttpServletRequestWrapper implements ServletOrJdiscHttpRequest { + public static final String JDISC_REQUEST_PRINCIPAL = "jdisc.request.principal"; private final HttpServletRequest request; private final HeaderFields headerFields; @@ -252,4 +254,14 @@ public class ServletRequest extends HttpServletRequestWrapper implements Servlet public long getConnectedAt(TimeUnit unit) { return unit.convert(connectedAt, TimeUnit.MILLISECONDS); } + + @Override + public Principal getUserPrincipal() { + // NOTE: The principal from the underlying servlet request is ignored. JDisc filters are the source-of-truth. + return (Principal) request.getAttribute(JDISC_REQUEST_PRINCIPAL); + } + + public void setUserPrincipal(Principal principal) { + request.setAttribute(JDISC_REQUEST_PRINCIPAL, principal); + } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreConfigurator.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreConfigurator.java new file mode 100644 index 00000000000..fb0a5869bb3 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreConfigurator.java @@ -0,0 +1,95 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl; + +import com.google.inject.Inject; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.SecretStore; +import com.yahoo.jdisc.http.ssl.pem.PemSslKeyStore; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +public class DefaultSslKeyStoreConfigurator implements SslKeyStoreConfigurator { + + private static final Logger log = Logger.getLogger(DefaultSslKeyStoreConfigurator.class.getName()); + + private final SecretStore secretStore; + private final ConnectorConfig.Ssl config; + + @Inject + public DefaultSslKeyStoreConfigurator(ConnectorConfig config, SecretStore secretStore) { + validateConfig(config.ssl()); + this.secretStore = secretStore; + this.config = config.ssl(); + } + + private static void validateConfig(ConnectorConfig.Ssl config) { + if (!config.enabled()) return; + switch (config.keyStoreType()) { + case JKS: + validateJksConfig(config); + break; + case PEM: + validatePemConfig(config); + break; + } + } + + @Override + public void configure(SslKeyStoreContext context) { + if (!config.enabled()) return; + switch (config.keyStoreType()) { + case JKS: + context.updateKeyStore(config.keyStorePath(), "JKS", secretStore.getSecret(config.keyDbKey())); + break; + case PEM: + context.updateKeyStore(createPemKeyStore(config.pemKeyStore())); + break; + } + } + + private static void validateJksConfig(ConnectorConfig.Ssl ssl) { + if (!ssl.pemKeyStore().keyPath().isEmpty() || ! ssl.pemKeyStore().certificatePath().isEmpty()) { + throw new IllegalArgumentException("pemKeyStore attributes can not be set when keyStoreType is JKS."); + } + if (ssl.keyDbKey().isEmpty()) { + throw new IllegalArgumentException("Missing password for JKS keystore"); + } + } + + private static void validatePemConfig(ConnectorConfig.Ssl ssl) { + if (! ssl.keyStorePath().isEmpty()) { + throw new IllegalArgumentException("keyStorePath can not be set when keyStoreType is PEM"); + } + if (!ssl.keyDbKey().isEmpty()) { + // TODO Make an error once there are separate passwords for truststore and keystore + log.warning("Encrypted PEM key stores are not supported. Password is only applied to truststore"); + } + if (ssl.pemKeyStore().certificatePath().isEmpty()) { + throw new IllegalArgumentException("Missing certificate path."); + } + if (ssl.pemKeyStore().keyPath().isEmpty()) { + throw new IllegalArgumentException("Missing key path."); + } + } + + private static KeyStore createPemKeyStore(ConnectorConfig.Ssl.PemKeyStore pemKeyStore) { + try { + Path certificatePath = Paths.get(pemKeyStore.certificatePath()); + Path keyPath = Paths.get(pemKeyStore.keyPath()); + return new PemSslKeyStore(certificatePath, keyPath).loadJavaKeyStore(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (Exception e) { + throw new RuntimeException("Failed setting up key store for " + pemKeyStore.keyPath() + ", " + pemKeyStore.certificatePath(), e); + } + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreContext.java new file mode 100644 index 00000000000..44a9c606576 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/DefaultSslKeyStoreContext.java @@ -0,0 +1,51 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl; + +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import java.security.KeyStore; +import java.util.function.Consumer; + +/** + * @author bjorncs + */ +public class DefaultSslKeyStoreContext implements SslKeyStoreContext { + + private final SslContextFactory sslContextFactory; + + public DefaultSslKeyStoreContext(SslContextFactory sslContextFactory) { + this.sslContextFactory = sslContextFactory; + } + + @Override + public void updateKeyStore(KeyStore keyStore) { + updateKeyStore(keyStore, null); + } + + @Override + public void updateKeyStore(KeyStore keyStore, String password) { + updateKeyStore(sslContextFactory -> { + sslContextFactory.setKeyStore(keyStore); + if (password != null) { + sslContextFactory.setKeyStorePassword(password); + } + }); + } + + @Override + public void updateKeyStore(String keyStorePath, String keyStoreType, String keyStorePassword) { + updateKeyStore(sslContextFactory -> { + sslContextFactory.setKeyStorePath(keyStorePath); + sslContextFactory.setKeyStoreType(keyStoreType); + sslContextFactory.setKeyStorePassword(keyStorePassword); + }); + } + + private void updateKeyStore(Consumer<SslContextFactory> reloader) { + try { + sslContextFactory.reload(reloader); + } catch (Exception e) { + throw new RuntimeException("Could not update keystore: " + e.getMessage(), e); + } + } +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/JKSKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/JKSKeyStore.java deleted file mode 100644 index ce3c6f6ea4a..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/JKSKeyStore.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; - -/** - * @author tonytv - */ -public class JKSKeyStore extends SslKeyStore { - - private static final String keyStoreType = "JKS"; - private final Path keyStoreFile; - - public JKSKeyStore(Path keyStoreFile) { - this.keyStoreFile = keyStoreFile; - } - - @Override - public KeyStore loadJavaKeyStore() throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { - try(InputStream stream = Files.newInputStream(keyStoreFile)) { - KeyStore keystore = KeyStore.getInstance(keyStoreType); - keystore.load(stream, getKeyStorePassword().map(String::toCharArray).orElse(null)); - return keystore; - } - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ReaderForPath.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ReaderForPath.java deleted file mode 100644 index b04d91d7403..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ReaderForPath.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import java.io.Reader; -import java.nio.file.Path; - -/** - * A reader along with the path used to construct it. - * - * @author tonytv - */ -public final class ReaderForPath { - - public final Reader reader; - public final Path path; - - public ReaderForPath(Reader reader, Path path) { - this.reader = reader; - this.path = path; - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStore.java deleted file mode 100644 index 1201bb08afc..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStore.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; - -import java.io.IOException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.util.Optional; - -/** - * - * @author <a href="mailto:charlesk@yahoo-inc.com">Charles Kim</a> - */ -public abstract class SslKeyStore { - - private Optional<String> keyStorePassword = Optional.empty(); - - public Optional<String> getKeyStorePassword() { - return keyStorePassword; - } - - public void setKeyStorePassword(String keyStorePassword) { - this.keyStorePassword = Optional.of(keyStorePassword); - } - - public abstract KeyStore loadJavaKeyStore() throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException; - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreConfigurator.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreConfigurator.java new file mode 100644 index 00000000000..619f4a636ed --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreConfigurator.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl; + +/** + * An interface for an component that can configure an {@link SslKeyStoreContext}. The implementor can assume that + * the {@link SslKeyStoreContext} instance is thread-safe and be updated at any time + * during and after the call to{@link #configure(SslKeyStoreContext)}. + * Modifying the {@link SslKeyStoreContext} instance will trigger a hot reload of the keystore in JDisc. + * + * @author bjorncs + */ +public interface SslKeyStoreConfigurator { + void configure(SslKeyStoreContext context); +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreContext.java new file mode 100644 index 00000000000..2a25f6d78b5 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslKeyStoreContext.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl; + +import java.security.KeyStore; + +/** + * An interface to update the keystore in JDisc. Any update will trigger a hot reload and new connections will + * immediately see the new certificate chain. + * + * @author bjorncs + */ +public interface SslKeyStoreContext { + void updateKeyStore(KeyStore keyStore); + void updateKeyStore(KeyStore keyStore, String password); + void updateKeyStore(String keyStorePath, String keyStoreType, String keyStorePassword); +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java index c47d36991d4..5f817d4cfc2 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/package-info.java @@ -1,4 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ @ExportPackage package com.yahoo.jdisc.http.ssl; + import com.yahoo.osgi.annotation.ExportPackage; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java index 21272f202ea..b52e923662f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java @@ -2,7 +2,6 @@ package com.yahoo.jdisc.http.ssl.pem; import com.google.common.base.Preconditions; -import com.yahoo.jdisc.http.ssl.ReaderForPath; import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; @@ -16,9 +15,13 @@ import javax.annotation.concurrent.GuardedBy; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.Key; +import java.security.KeyStore; import java.security.KeyStore.LoadStoreParameter; -import java.security.KeyStore.ProtectionParameter; import java.security.KeyStoreException; import java.security.KeyStoreSpi; import java.security.NoSuchAlgorithmException; @@ -58,10 +61,6 @@ public class PemKeyStore extends KeyStoreSpi { @GuardedBy("this") private final Map<String, Certificate> aliasToCertificate = new LinkedHashMap<>(); - - public PemKeyStore() {} - - /** * The user is responsible for closing any readers given in the parameter. */ @@ -287,30 +286,51 @@ public class PemKeyStore extends KeyStoreSpi { } } - public static class PemLoadStoreParameter implements LoadStoreParameter { - private PemLoadStoreParameter() {} + // A reader along with the path used to construct it. + private static class ReaderForPath { + final Reader reader; + final Path path; - @Override - public ProtectionParameter getProtectionParameter() { - return null; + private ReaderForPath(Reader reader, Path path) { + this.reader = reader; + this.path = path; + } + + static ReaderForPath of(Path path) { + try { + return new ReaderForPath(Files.newBufferedReader(path), path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } - public static final class KeyStoreLoadParameter extends PemLoadStoreParameter { - public final ReaderForPath certificateReader; - public final ReaderForPath keyReader; + static class TrustStoreLoadParameter implements KeyStore.LoadStoreParameter { + final ReaderForPath certificateReader; - public KeyStoreLoadParameter(ReaderForPath certificateReader, ReaderForPath keyReader) { - this.certificateReader = certificateReader; - this.keyReader = keyReader; + TrustStoreLoadParameter(Path certificateReader) { + this.certificateReader = ReaderForPath.of(certificateReader); + } + + @Override + public KeyStore.ProtectionParameter getProtectionParameter() { + return null; } } - public static final class TrustStoreLoadParameter extends PemLoadStoreParameter { - public final ReaderForPath certificateReader; + static class KeyStoreLoadParameter implements KeyStore.LoadStoreParameter { + final ReaderForPath certificateReader; + final ReaderForPath keyReader; + + KeyStoreLoadParameter(Path certificateReader, Path keyReader) { + this.certificateReader = ReaderForPath.of(certificateReader); + this.keyReader = ReaderForPath.of(keyReader); + } - public TrustStoreLoadParameter(ReaderForPath certificateReader) { - this.certificateReader = certificateReader; + @Override + public KeyStore.ProtectionParameter getProtectionParameter() { + return null; } } + } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStoreProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStoreProvider.java deleted file mode 100644 index c1fcf8c33bf..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStoreProvider.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl.pem; - -import java.security.Provider; - -/** - * @author Tony Vaagenes - */ -public class PemKeyStoreProvider extends Provider { - - public static final String name = "PEMKeyStoreProvider"; - public static final double version = 1; - public static final String description = "Provides PEM keystore support"; - - public PemKeyStoreProvider() { - super(name, version, description); - putService(new Service(this, "KeyStore", "PEM", PemKeyStore. class.getName(), PemKeyStore.aliases, PemKeyStore.attributes)); - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java index bf91f0eb259..2ae1894a8d4 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.ssl.pem; -import com.yahoo.jdisc.http.ssl.SslKeyStore; import com.yahoo.jdisc.http.ssl.pem.PemKeyStore.KeyStoreLoadParameter; -import com.yahoo.jdisc.http.ssl.pem.PemKeyStore.PemLoadStoreParameter; import com.yahoo.jdisc.http.ssl.pem.PemKeyStore.TrustStoreLoadParameter; import java.io.IOException; +import java.nio.file.Path; import java.security.KeyStore; +import java.security.KeyStore.LoadStoreParameter; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; +import java.security.Provider; import java.security.Security; import java.security.cert.CertificateException; @@ -17,37 +18,46 @@ import java.security.cert.CertificateException; * Responsible for creating pem key stores. * * @author Tony Vaagenes + * @author bjorncs */ -public class PemSslKeyStore extends SslKeyStore { +public class PemSslKeyStore { static { Security.addProvider(new PemKeyStoreProvider()); } - private static final String keyStoreType = "PEM"; - private final PemLoadStoreParameter loadParameter; + private static final String KEY_STORE_TYPE = "PEM"; + + private final LoadStoreParameter loadParameter; private KeyStore keyStore; - public PemSslKeyStore(KeyStoreLoadParameter loadParameter) { - this.loadParameter = loadParameter; + public PemSslKeyStore(Path certificatePath, Path keyPath) { + this.loadParameter = new KeyStoreLoadParameter(certificatePath, keyPath); } - public PemSslKeyStore(TrustStoreLoadParameter loadParameter) { - this.loadParameter = loadParameter; + public PemSslKeyStore(Path certificatePath) { + this.loadParameter = new TrustStoreLoadParameter(certificatePath); } - @Override - public KeyStore loadJavaKeyStore() throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { - if (getKeyStorePassword().isPresent()) { - throw new UnsupportedOperationException("PEM key store with password is currently not supported. Please file a feature request."); - } - - //cached since Reader(in loadParameter) can only be used one time. + public KeyStore loadJavaKeyStore() + throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException { if (keyStore == null) { - keyStore = KeyStore.getInstance(keyStoreType); + keyStore = KeyStore.getInstance(KEY_STORE_TYPE); keyStore.load(loadParameter); } return keyStore; } + private static class PemKeyStoreProvider extends Provider { + + static final String NAME = "PEMKeyStoreProvider"; + static final double VERSION = 1; + static final String DESCRIPTION = "Provides PEM keystore support"; + + PemKeyStoreProvider() { + super(NAME, VERSION, DESCRIPTION); + putService(new Service(this, "KeyStore", "PEM", PemKeyStore. class.getName(), PemKeyStore.aliases, PemKeyStore.attributes)); + } + } + } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/ChunkReader.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/ChunkReader.java deleted file mode 100644 index a550a013a3b..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/ChunkReader.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.test; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> - */ -public class ChunkReader { - - private static final Pattern CONTENT_LENGTH = Pattern.compile(".+^content-length: (\\d+)$.*", - Pattern.CASE_INSENSITIVE | - Pattern.MULTILINE | - Pattern.DOTALL); - private static final Pattern CHUNKED_ENCODING = Pattern.compile(".+^transfer-encoding: chunked$.*", - Pattern.CASE_INSENSITIVE | - Pattern.MULTILINE | - Pattern.DOTALL); - private final InputStream in; - private StringBuilder reading = new StringBuilder(); - private boolean readingHeader = true; - - public ChunkReader(InputStream in) { - this.in = in; - } - - public boolean isEndOfContent() throws IOException { - if (in.available() != 0) { - StringBuilder sb = new StringBuilder(); - sb.append(in.available()).append(": "); - for(int c = in.read(); c != -1; c = in.read()) { - sb.append('\''); - sb.append(c); - sb.append("' "); - } - throw new IllegalStateException("This is not the end '" + sb.toString()); - } - return in.available() == 0; - } - - public String readChunk() throws IOException { - while (true) { - String ret = removeNextChunk(); - if (ret != null) { - return ret; - } - readFromStream(); - } - } - - private String readContent(int length) throws IOException { - while (reading.length() < length) { - readFromStream(); - } - return splitReadBuffer(length); - } - - private void readFromStream() throws IOException { - byte[] buf = new byte[4096]; - try { - while (!Thread.currentThread().isInterrupted()) { - int len = in.read(buf, 0, buf.length); - if (len < 0) { - throw new IOException("Socket is closed."); - } - if (len > 0) { - reading.append(StandardCharsets.UTF_8.decode(ByteBuffer.wrap(buf, 0, len))); - break; - } - Thread.sleep(10); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private String removeNextChunk() throws IOException { - if (readingHeader) { - int pos = reading.indexOf("\r\n\r\n"); - if (pos < 0) { - return null; - } - String ret = splitReadBuffer(pos + 4); - Matcher m = CONTENT_LENGTH.matcher(ret); - if (m.matches()) { - ret += readContent(Integer.valueOf(m.group(1))); - } - readingHeader = !CHUNKED_ENCODING.matcher(ret).matches(); - return ret; - } else if (reading.indexOf("0\r\n") == 0) { - int pos = reading.indexOf("\r\n\r\n", 1); - if (pos < 0) { - return null; - } - readingHeader = true; - return splitReadBuffer(pos + 4); - } else { - int pos = reading.indexOf("\r\n"); - if (pos < 0) { - return null; - } - pos = reading.indexOf("\r\n", pos + 2); - if (pos < 0) { - return null; - } - return splitReadBuffer(pos + 2); - } - } - - private String splitReadBuffer(int pos) { - String ret = reading.substring(0, pos); - if (pos < reading.length()) { - reading = new StringBuilder(reading.substring(pos)); - } else { - reading = new StringBuilder(); - } - return ret; - } -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/FilterTestDriver.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/FilterTestDriver.java deleted file mode 100644 index 1532bc65bdf..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/FilterTestDriver.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.test; - -import com.yahoo.jdisc.Request; -import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.application.BindingRepository; -import com.yahoo.jdisc.handler.AbstractRequestHandler; -import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.jdisc.handler.ResponseDispatch; -import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.jdisc.http.filter.RequestFilter; -import com.yahoo.jdisc.http.filter.ResponseFilter; - -import java.io.IOException; -import java.util.concurrent.Exchanger; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static com.yahoo.jdisc.http.test.ServerTestDriver.newFilterModule; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> - * - * TODO: dead code? - */ -public class FilterTestDriver { - - private final ServerTestDriver driver; - private final MyRequestHandler requestHandler; - - private FilterTestDriver(ServerTestDriver driver, MyRequestHandler requestHandler) { - this.driver = driver; - this.requestHandler = requestHandler; - } - - public boolean close() throws IOException { - return driver.close(); - } - - public HttpRequest filterRequest(String request) throws IOException, TimeoutException, InterruptedException { - driver.client().writeRequest(request); - return (HttpRequest)requestHandler.exchanger.exchange(null, 60, TimeUnit.SECONDS); - } - - public static FilterTestDriver newInstance(final BindingRepository<RequestFilter> requestFilters, - final BindingRepository<ResponseFilter> responseFilters) - throws IOException { - MyRequestHandler handler = new MyRequestHandler(); - return new FilterTestDriver(ServerTestDriver.newInstance(handler, - newFilterModule(requestFilters, responseFilters)), - handler); - } - - private static class MyRequestHandler extends AbstractRequestHandler { - - final Exchanger<Request> exchanger = new Exchanger<>(); - - @Override - public ContentChannel handleRequest(Request request, ResponseHandler handler) { - ResponseDispatch.newInstance(Response.Status.OK).dispatch(handler); - try { - exchanger.exchange(request); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return null; - } - } -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/RemoteClient.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/RemoteClient.java deleted file mode 100644 index dd6033c9975..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/RemoteClient.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.test; - -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; -import com.yahoo.jdisc.http.ssl.SslContextFactory; -import com.yahoo.jdisc.http.ssl.SslKeyStore; - -import javax.net.ssl.SSLContext; -import java.io.IOException; -import java.net.Socket; -import java.nio.charset.StandardCharsets; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> - */ -public class RemoteClient extends ChunkReader { - - private final Socket socket; - - private RemoteClient(Socket socket) throws IOException { - super(socket.getInputStream()); - this.socket = socket; - } - - public void close() throws IOException { - socket.close(); - } - - public void writeRequest(String request) throws IOException { - socket.getOutputStream().write(request.getBytes(StandardCharsets.UTF_8)); - } - - public static RemoteClient newInstance(JettyHttpServer server) throws IOException { - return newInstance(server.getListenPort()); - } - - public static RemoteClient newInstance(int listenPort) throws IOException { - return new RemoteClient(new Socket("localhost", listenPort)); - } - - public static RemoteClient newSslInstance(int listenPort, SslKeyStore sslKeyStore) throws IOException { - SSLContext ctx = SslContextFactory.newInstanceFromTrustStore(sslKeyStore).getServerSSLContext(); - if (ctx == null) { - throw new RuntimeException("Failed to create socket with SSLContext."); - } - return new RemoteClient(ctx.getSocketFactory().createSocket("localhost", listenPort)); - } - - public static RemoteClient newSslInstance(JettyHttpServer server, SslKeyStore keyStore) throws IOException { - return newSslInstance(server.getListenPort(), keyStore); - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/RemoteServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/RemoteServer.java deleted file mode 100644 index 62b4bb306ed..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/RemoteServer.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.test; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> - */ -public class RemoteServer implements Runnable { - - private final Thread thread = new Thread(this, "RemoteServer@" + System.identityHashCode(this)); - private final LinkedBlockingQueue<Socket> clients = new LinkedBlockingQueue<>(); - private final ServerSocket server; - - private RemoteServer(int listenPort) throws IOException { - this.server = new ServerSocket(listenPort); - } - - @Override - public void run() { - try { - while (!Thread.interrupted()) { - Socket client = server.accept(); - if (client != null) { - clients.add(client); - } - } - } catch (IOException e) { - if (!server.isClosed()) { - e.printStackTrace(); - } - } - } - - public URI newRequestUri(String uri) { - return newRequestUri(URI.create(uri)); - } - - public URI newRequestUri(URI uri) { - URI serverUri = connectionSpec(); - try { - return new URI(serverUri.getScheme(), serverUri.getUserInfo(), serverUri.getHost(), - serverUri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - } - - public URI connectionSpec() { - return URI.create("http://localhost:" + server.getLocalPort() + "/"); - } - - public Connection awaitConnection(int timeout, TimeUnit unit) throws InterruptedException, IOException { - Socket client = clients.poll(timeout, unit); - if (client == null) { - return null; - } - return new Connection(client); - } - - public boolean close(int timeout, TimeUnit unit) { - try { - server.close(); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - try { - thread.join(unit.toMillis(timeout)); - } catch (InterruptedException e) { - return false; - } - return !thread.isAlive(); - } - - public static RemoteServer newInstance() throws IOException { - RemoteServer ret = new RemoteServer(0); - ret.thread.start(); - return ret; - } - - public static class Connection extends ChunkReader { - - private final Socket socket; - private final PrintWriter out; - - private Connection(Socket socket) throws IOException { - super(socket.getInputStream()); - this.socket = socket; - this.out = new PrintWriter(new OutputStreamWriter(socket.getOutputStream())); - } - - public void writeChunk(String chunk) { - out.print(chunk); - } - - public void close() throws IOException { - out.close(); - socket.close(); - } - } -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/ServerTestDriver.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/ServerTestDriver.java deleted file mode 100644 index 03e2257ce70..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/test/ServerTestDriver.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.test; - -import com.google.inject.AbstractModule; -import com.google.inject.Module; -import com.google.inject.TypeLiteral; -import com.yahoo.jdisc.application.BindingRepository; -import com.yahoo.jdisc.application.ContainerActivator; -import com.yahoo.jdisc.application.ContainerBuilder; -import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.jdisc.http.filter.RequestFilter; -import com.yahoo.jdisc.http.filter.ResponseFilter; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; -import com.yahoo.jdisc.http.ssl.SslKeyStore; -import com.yahoo.jdisc.test.TestDriver; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> - */ -public class ServerTestDriver { - - private final TestDriver driver; - private final JettyHttpServer server; - private final RemoteClient client; - - private ServerTestDriver(TestDriver driver, JettyHttpServer server, RemoteClient client) { - this.driver = driver; - this.server = server; - this.client = client; - } - - public boolean close() throws IOException { - client.close(); - server.close(); - server.release(); - return driver.close(); - } - - public TestDriver parent() { - return driver; - } - - public ContainerActivator containerActivator() { - return driver; - } - - public JettyHttpServer server() { - return server; - } - - public RemoteClient client() { - return client; - } - - public HttpRequest newRequest(HttpRequest.Method method, String uri, HttpRequest.Version version) { - return HttpRequest.newServerRequest(driver, newRequestUri(uri), method, version); - } - - public URI newRequestUri(String uri) { - return newRequestUri(URI.create(uri)); - } - - public URI newRequestUri(URI uri) { - try { - return new URI("http", null, "locahost", - server.getListenPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - } - - public static ServerTestDriver newInstance(RequestHandler requestHandler, Module... guiceModules) throws IOException { - return newInstance(requestHandler, Arrays.asList(guiceModules)); - } - - public static ServerTestDriver newInstance(RequestHandler requestHandler, Iterable<Module> guiceModules) - throws IOException { - List<Module> lst = new LinkedList<>(); - lst.add(newDefaultModule()); - for (Module module : guiceModules) { - lst.add(module); - } - TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(lst.toArray(new Module[lst.size()])); - ContainerBuilder builder = driver.newContainerBuilder(); - builder.serverBindings().bind("*://*/*", requestHandler); - JettyHttpServer server = builder.guiceModules().getInstance(JettyHttpServer.class); - return newInstance(null, driver, builder, server); - } - - private static ServerTestDriver newInstance(SslKeyStore clientTrustStore, TestDriver driver, ContainerBuilder builder, - JettyHttpServer server) throws IOException { - builder.serverProviders().install(server); - driver.activateContainer(builder); - try { - server.start(); - } catch (RuntimeException e) { - server.release(); - driver.close(); - throw e; - } - RemoteClient client; - if (clientTrustStore == null) { - client = RemoteClient.newInstance(server); - } else { - client = RemoteClient.newSslInstance(server, clientTrustStore); - } - return new ServerTestDriver(driver, server, client); - } - - public static Module newDefaultModule() { - return new AbstractModule() { - - @Override - protected void configure() { - bind(new TypeLiteral<BindingRepository<RequestFilter>>() { }) - .toInstance(new BindingRepository<>()); - bind(new TypeLiteral<BindingRepository<ResponseFilter>>() { }) - .toInstance(new BindingRepository<>()); - } - }; - } - - public static Module newFilterModule(final BindingRepository<RequestFilter> requestFilters, - final BindingRepository<ResponseFilter> responseFilters) { - return new AbstractModule() { - - @Override - protected void configure() { - if (requestFilters != null) { - bind(new TypeLiteral<BindingRepository<RequestFilter>>() { }).toInstance(requestFilters); - } - if (responseFilters != null) { - bind(new TypeLiteral<BindingRepository<ResponseFilter>>() { }).toInstance(responseFilters); - } - } - }; - } -} diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 8d709cb8ab1..59893753bea 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -45,8 +45,9 @@ ssl.enabled bool default=false # The name of the key to the password to the key store if in the secret store, if JKS is used. # Must be empty with PEM -# By default this is also used to look up the password to the trust store. +# By default this is also used to look up the password to the trust store. ssl.keyDbKey string default="" +# TODO Rename keyDbKey to keyStorePassword after introducing custom services.xml syntax # Names of protocols to exclude. ssl.excludeProtocol[].name string @@ -74,9 +75,12 @@ ssl.trustStoreType enum { JKS } default=JKS # JKS only - the path to the truststore. ssl.trustStorePath string default="" +# TODO Add separate config for truststore password + # Whether we should use keyDbKey as password to the trust store (true, default), # or use no password with the trust store (false) ssl.useTrustStorePassword bool default=true +# TODO Fix broken semantics with truststore and keystore password in Vespa 7 / Vespa 8 # The algorithm name used by the KeyManagerFactory. ssl.sslKeyManagerFactoryAlgorithm string default="SunX509" diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/AssertFile.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/AssertFile.java deleted file mode 100644 index 24739fc47a6..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/AssertFile.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.MatcherAssert.assertThat; - - -/** - * @author <a href="mailto:apurvak@yahoo-inc.com">Apurva Kumar</a> - */ - -public class AssertFile { - - public static void assertContains(File logFile, String expected) throws IOException { - String s = new String( - Files.readAllBytes(Paths.get(logFile.getAbsolutePath())), - StandardCharsets.UTF_8); - assertThat(s, containsString(expected)); - } - - public static void assertNotContains(File logFile, String expected) throws IOException { - String s = new String( - Files.readAllBytes(Paths.get(logFile.getAbsolutePath())), - StandardCharsets.UTF_8); - assertThat(s, not(containsString(expected))); - } -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/AssertHttp.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/AssertHttp.java deleted file mode 100644 index dfc9550191c..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/AssertHttp.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http; - -import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.jdisc.http.test.RemoteClient; -import com.yahoo.jdisc.http.test.ServerTestDriver; - -import java.io.IOException; -import java.util.Arrays; -import java.util.regex.Pattern; - -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertTrue; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> - */ -public abstract class AssertHttp { - - public static void assertChunk(String expected, String actual) { - if (expected.startsWith("HTTP/1.")) { - expected = sortChunk(expected); - actual = sortChunk(actual); - } - Pattern pattern = Pattern.compile(expected, Pattern.DOTALL | Pattern.MULTILINE); - if (pattern.matcher(actual).matches()) { - return; - } - assertEquals(expected, actual); - } - - public static void assertResponse(RequestHandler requestHandler, String request, - String... expectedChunks) throws IOException { - ServerTestDriver driver = ServerTestDriver.newInstance(requestHandler); - assertResponse(driver, request, expectedChunks); - assertTrue(driver.close()); - } - - public static void assertResponse(ServerTestDriver driver, String request, String... expectedChunks) - throws IOException { - assertResponse(driver.client(), request, expectedChunks); - } - - public static void assertResponse(RemoteClient client, String request, String... expectedChunks) - throws IOException { - client.writeRequest(request); - for (String expected : expectedChunks) { - assertChunk(expected, client.readChunk()); - } - } - - private static String sortChunk(String chunk) { - String[] lines = chunk.split("\r\n"); - if (lines.length > 2) { - int prev = 1, next = 2; - for ( ; next < lines.length && !lines[next].isEmpty(); ++next) { - if (!Character.isLetterOrDigit(lines[next].charAt(0))) { - Arrays.sort(lines, prev, next); - prev = next + 1; - } - } - if (prev < next) { - Arrays.sort(lines, prev, next); - } - } - StringBuilder out = new StringBuilder(); - for (String line : lines) { - out.append(line).append("\r\n"); - } - return out.toString(); - } -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/DummyMetricManager.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/DummyMetricManager.java deleted file mode 100644 index 100cc19477d..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/DummyMetricManager.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http; - -import com.google.inject.AbstractModule; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.application.MetricConsumer; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author <a href="mailto:ssameer@yahoo-inc.com">ssameer</a> - * Date: 2/15/13 - * Time: 11:49 AM - */ -public class DummyMetricManager extends AbstractModule implements MetricConsumer { - - private final Map<String, Integer> metrics = new HashMap<>(); - private Map<String, ?> lastContextDimensions; - - @Override - protected void configure() { - bind(MetricConsumer.class).toInstance(this); - } - - @Override - public void add(String key, Number val, Metric.Context ctx) { - synchronized (metrics) { - metrics.put(key, get(key) + val.intValue()); - } - } - - @Override - public void set(String key, Number val, Metric.Context ctx) { - synchronized (metrics) { - metrics.put(key, val.intValue()); - } - } - - @Override - public Metric.Context createContext(Map<String, ?> dimensions) { - lastContextDimensions = dimensions; - return new Metric.Context() { }; - } - - public Map<String, ?> getLastContextDimensions() { - return lastContextDimensions; - } - - public int get(String key) { - Integer val; - synchronized (metrics) { - val = metrics.get(key); - } - return val != null ? val : 0; - } - -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/JksKeyStore.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/JksKeyStore.java new file mode 100644 index 00000000000..1c7a917c688 --- /dev/null +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/JksKeyStore.java @@ -0,0 +1,41 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; + +/** + * @author Tony Vaagenes + * @author bjorncs + */ +public class JksKeyStore { + + private static final String KEY_STORE_TYPE = "JKS"; + + private final Path keyStoreFile; + private final String keyStorePassword; + + public JksKeyStore(Path keyStoreFile) { + this(keyStoreFile, null); + } + + public JksKeyStore(Path keyStoreFile, String keyStorePassword) { + this.keyStoreFile = keyStoreFile; + this.keyStorePassword = keyStorePassword; + } + + public String getKeyStorePassword() { + return keyStorePassword; + } + + public KeyStore loadJavaKeyStore() throws Exception { + try(InputStream stream = Files.newInputStream(keyStoreFile)) { + KeyStore keystore = KeyStore.getInstance(KEY_STORE_TYPE); + keystore.load(stream, keyStorePassword != null ? keyStorePassword.toCharArray() : null); + return keystore; + } + } + +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactory.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/SslContextFactory.java index 0f39c132b67..d86516df453 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactory.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/SslContextFactory.java @@ -1,19 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl; +package com.yahoo.jdisc.http; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; -import java.io.IOException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; -import java.security.cert.CertificateException; import java.util.logging.Level; import java.util.logging.Logger; /** - * @author <a href="mailto:charlesk@yahoo-inc.com">Charles Kim</a> + * @author Charles Kim */ public class SslContextFactory { @@ -30,16 +25,16 @@ public class SslContextFactory { return this.sslContext; } - public static SslContextFactory newInstanceFromTrustStore(SslKeyStore trustStore) { + public static SslContextFactory newInstanceFromTrustStore(JksKeyStore trustStore) { return newInstance(DEFAULT_ALGORITHM, DEFAULT_PROTOCOL, null, trustStore); } - public static SslContextFactory newInstance(SslKeyStore trustStore, SslKeyStore keyStore) { + public static SslContextFactory newInstance(JksKeyStore trustStore, JksKeyStore keyStore) { return newInstance(DEFAULT_ALGORITHM, DEFAULT_PROTOCOL, keyStore, trustStore); } public static SslContextFactory newInstance(String sslAlgorithm, String sslProtocol, - SslKeyStore keyStore, SslKeyStore trustStore) { + JksKeyStore keyStore, JksKeyStore trustStore) { log.fine("Configuring SSLContext..."); log.fine("Using " + sslAlgorithm + " algorithm."); try { @@ -58,15 +53,14 @@ public class SslContextFactory { /** * Used for the key store, which contains the SSL cert and private key. */ - public static javax.net.ssl.KeyManager[] getKeyManagers(SslKeyStore keyStore, - String sslAlgorithm) - throws NoSuchAlgorithmException, CertificateException, IOException, UnrecoverableKeyException, - KeyStoreException { + public static javax.net.ssl.KeyManager[] getKeyManagers(JksKeyStore keyStore, + String sslAlgorithm) throws Exception { KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(sslAlgorithm); + String keyStorePassword = keyStore.getKeyStorePassword(); keyManagerFactory.init( keyStore.loadJavaKeyStore(), - keyStore.getKeyStorePassword().map(String::toCharArray).orElse(null)); + keyStorePassword != null ? keyStorePassword.toCharArray() : null); log.fine("KeyManagerFactory initialized with keystore"); return keyManagerFactory.getKeyManagers(); } @@ -75,9 +69,9 @@ public class SslContextFactory { * Used for the trust store, which contains certificates from other parties that you expect to communicate with, * or from Certificate Authorities that you trust to identify other parties. */ - public static javax.net.ssl.TrustManager[] getTrustManagers(SslKeyStore trustStore, + public static javax.net.ssl.TrustManager[] getTrustManagers(JksKeyStore trustStore, String sslAlgorithm) - throws NoSuchAlgorithmException, CertificateException, IOException, KeyStoreException { + throws Exception { TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(sslAlgorithm); trustManagerFactory.init(trustStore.loadJavaKeyStore()); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java index 1200a06be2c..0d8f433cc39 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/guiceModules/ConnectorFactoryRegistryModule.java @@ -11,6 +11,7 @@ import com.yahoo.jdisc.http.ConnectorConfig.Builder; import com.yahoo.jdisc.http.SecretStore; import com.yahoo.jdisc.http.server.jetty.ConnectorFactory; import com.yahoo.jdisc.http.server.jetty.TestDrivers; +import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; /** * Guice module for test ConnectorFactories @@ -46,7 +47,9 @@ public class ConnectorFactoryRegistryModule implements Module { private static class StaticKeyDbConnectorFactory extends ConnectorFactory { public StaticKeyDbConnectorFactory(ConnectorConfig connectorConfig) { - super(connectorConfig, new MockSecretStore()); + super(connectorConfig, + new MockSecretStore(), + new DefaultSslKeyStoreConfigurator(connectorConfig, new MockSecretStore())); } } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java index 7a03d805864..781bc6a7b5f 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java @@ -1,33 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.google.common.collect.ImmutableMap; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.SecretStore; -import com.yahoo.jdisc.http.ssl.ReaderForPath; -import com.yahoo.jdisc.http.ssl.SslContextFactory; -import com.yahoo.jdisc.http.ssl.SslKeyStore; -import com.yahoo.jdisc.http.ssl.pem.PemKeyStore; -import com.yahoo.jdisc.http.ssl.pem.PemSslKeyStore; +import com.yahoo.jdisc.http.ssl.DefaultSslKeyStoreConfigurator; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.AbstractHandler; import org.testng.annotations.Test; -import javax.net.ssl.SSLContext; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.InetSocketAddress; -import java.nio.channels.FileChannel; import java.nio.channels.ServerSocketChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.Collections; import java.util.Map; import static com.yahoo.jdisc.http.ConnectorConfig.Ssl; @@ -36,7 +24,7 @@ import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType.Enum.PEM; import static org.hamcrest.CoreMatchers.equalTo; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class ConnectorFactoryTest { @@ -50,8 +38,7 @@ public class ConnectorFactoryTest { .pemKeyStore( new Ssl.PemKeyStore.Builder() .keyPath("nonEmpty")))); - - ConnectorFactory willThrowException = new ConnectorFactory(config, new ThrowingSecretStore()); + ConnectorFactory willThrowException = createConnectorFactory(config); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -62,18 +49,17 @@ public class ConnectorFactoryTest { .enabled(true) .keyStoreType(PEM) .keyStorePath("nonEmpty"))); - - ConnectorFactory willThrowException = new ConnectorFactory(config, new ThrowingSecretStore()); + ConnectorFactory willThrowException = createConnectorFactory(config); } @Test public void requireThatNoPreBoundChannelWorks() throws Exception { Server server = new Server(); try { - ConnectorFactory factory = new ConnectorFactory(new ConnectorConfig(new ConnectorConfig.Builder()), - new ThrowingSecretStore()); - ConnectorFactory.JDiscServerConnector connector = - (ConnectorFactory.JDiscServerConnector)factory.createConnector(new DummyMetric(), server, null, Collections.emptyMap()); + ConnectorConfig config = new ConnectorConfig(new ConnectorConfig.Builder()); + ConnectorFactory factory = createConnectorFactory(config); + JDiscServerConnector connector = + (JDiscServerConnector)factory.createConnector(new DummyMetric(), server, null); server.addConnector(connector); server.setHandler(new HelloWorldHandler()); server.start(); @@ -98,8 +84,10 @@ public class ConnectorFactoryTest { ServerSocketChannel serverChannel = ServerSocketChannel.open(); serverChannel.socket().bind(new InetSocketAddress(0)); - ConnectorFactory factory = new ConnectorFactory(new ConnectorConfig(new ConnectorConfig.Builder()), new ThrowingSecretStore()); - ConnectorFactory.JDiscServerConnector connector = (ConnectorFactory.JDiscServerConnector) factory.createConnector(new DummyMetric(), server, serverChannel, Collections.emptyMap()); + ConnectorConfig config = new ConnectorConfig(new ConnectorConfig.Builder()); + ConnectorFactory factory = createConnectorFactory(config); + JDiscServerConnector connector = + (JDiscServerConnector) factory.createConnector(new DummyMetric(), server, serverChannel); server.addConnector(connector); server.setHandler(new HelloWorldHandler()); server.start(); @@ -117,61 +105,9 @@ public class ConnectorFactoryTest { } } - @Test - public void pre_bound_keystore_file_channels_are_used() throws Exception { - Path pemKeyStoreDirectory = Paths.get("src/test/resources/pem/"); - - Path certificateFile = pemKeyStoreDirectory.resolve("test.crt"); - Path privateKeyFile = pemKeyStoreDirectory.resolve("test.key"); - - Server server = new Server(); - try { - ServerSocketChannel serverChannel = ServerSocketChannel.open(); - serverChannel.socket().bind(new InetSocketAddress(0)); - - String fakeCertificatePath = "ensure-certificate-path-is-not-used-to-open-the-file"; - String fakeKeyPath = "ensure-key-path-is-not-used-to-open-the-file"; - - ConnectorConfig.Builder builder = new ConnectorConfig.Builder(); - builder.ssl( - new Ssl.Builder(). - enabled(true). - keyStoreType(PEM). - pemKeyStore(new Ssl.PemKeyStore.Builder(). - certificatePath(fakeCertificatePath). - keyPath(fakeKeyPath))); - - FileChannel certificateChannel = FileChannel.open(certificateFile, StandardOpenOption.READ); - FileChannel privateKeyChannel = FileChannel.open(privateKeyFile, StandardOpenOption.READ); - - Map<Path, FileChannel> keyStoreChannels = ImmutableMap.<Path, FileChannel>builder(). - put(Paths.get(fakeCertificatePath), certificateChannel). - put(Paths.get(fakeKeyPath), privateKeyChannel). - build(); - - - ConnectorFactory factory = new ConnectorFactory(new ConnectorConfig(builder), new ThrowingSecretStore()); - ConnectorFactory.JDiscServerConnector connector = (ConnectorFactory.JDiscServerConnector) factory.createConnector(new DummyMetric(), server, serverChannel, keyStoreChannels); - server.addConnector(connector); - server.setHandler(new HelloWorldHandler()); - server.start(); - - SslKeyStore trustStore = new PemSslKeyStore( - new PemKeyStore.TrustStoreLoadParameter( - new ReaderForPath(Files.newBufferedReader(certificateFile), certificateFile))); - - SSLContext clientSslContext = SslContextFactory.newInstanceFromTrustStore(trustStore).getServerSSLContext(); - SimpleHttpClient client = new SimpleHttpClient(clientSslContext, connector.getLocalPort(), false); - SimpleHttpClient.RequestExecutor ex = client.newGet("/ignored"); - SimpleHttpClient.ResponseValidator val = ex.execute(); - val.expectContent(equalTo("Hello world")); - } finally { - try { - server.stop(); - } catch (Exception e) { - //ignore - } - } + private static ConnectorFactory createConnectorFactory(ConnectorConfig config) { + ThrowingSecretStore secretStore = new ThrowingSecretStore(); + return new ConnectorFactory(config, secretStore, new DefaultSslKeyStoreConfigurator(config, secretStore)); } private static class HelloWorldHandler extends AbstractHandler { diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java index d588ace8268..80c1cb8b458 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java @@ -24,6 +24,7 @@ import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.net.URI; @@ -33,6 +34,8 @@ import java.util.Collections; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.regex.Pattern; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; @@ -49,13 +52,34 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ public class HttpServerConformanceTest extends ServerProviderConformanceTest { + private static final Logger log = Logger.getLogger(HttpServerConformanceTest.class.getName()); + private static final String REQUEST_CONTENT = "myRequestContent"; private static final String RESPONSE_CONTENT = "myResponseContent"; + @SuppressWarnings("LoggerInitializedWithForeignClass") + private static Logger httpRequestDispatchLogger = Logger.getLogger(HttpRequestDispatch.class.getName()); + private static Level httpRequestDispatchLoggerOriginalLevel; + + /* + * Reduce logging of every stack trace for {@link ServerProviderConformanceTest.ConformanceException} thrown. + * This makes the log more readable and the test faster as well. + */ + @BeforeClass + public static void reduceExcessiveLogging() { + httpRequestDispatchLoggerOriginalLevel = httpRequestDispatchLogger.getLevel(); + httpRequestDispatchLogger.setLevel(Level.SEVERE); + } + + @AfterClass + public static void restoreExcessiveLogging() { + httpRequestDispatchLogger.setLevel(httpRequestDispatchLoggerOriginalLevel); + } + @AfterClass public static void reportDiagnostics() { System.out.println( @@ -257,7 +281,7 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestContentWriteNondeterministicException() throws Throwable { - new TestRunner().expect(anyOf(success(), serverError())) + new TestRunner().expect(anyOf(success(), serverError(), successNoContent())) .execute(); } @@ -552,7 +576,7 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestContentCloseNondeterministicExceptionWithSyncFailure() throws Throwable { - new TestRunner().expect(anyOf(success(), serverError())) + new TestRunner().expect(anyOf(success(), successNoContent(), serverError())) .execute(); } @@ -784,7 +808,7 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { post.setProtocolVersion(client.requestVersion); request = post; } - System.out.println("executorService:" + log.fine(() -> "executorService:" + " .isShutDown()=" + executorService.isShutdown() + " .isTerminated()=" + executorService.isTerminated()); return executorService.submit(() -> client.delegate.execute(request)); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java index 30f0a5d4d6c..bcc23facd95 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java @@ -6,9 +6,8 @@ import com.google.inject.Module; import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.JKSKeyStore; -import com.yahoo.jdisc.http.ssl.SslContextFactory; -import com.yahoo.jdisc.http.ssl.SslKeyStore; +import com.yahoo.jdisc.http.SslContextFactory; +import com.yahoo.jdisc.http.JksKeyStore; import javax.net.ssl.SSLContext; import java.io.IOException; @@ -76,8 +75,9 @@ public class TestDriver { ConnectorConfig.Ssl sslConfig = builder.getInstance(ConnectorConfig.class).ssl(); if (!sslConfig.enabled()) return null; - SslKeyStore keyStore = new JKSKeyStore(Paths.get(sslConfig.keyStorePath())); - keyStore.setKeyStorePassword(builder.getInstance(Key.get(String.class, named("keyStorePassword")))); + JksKeyStore keyStore = new JksKeyStore( + Paths.get(sslConfig.keyStorePath()), + builder.getInstance(Key.get(String.class, named("keyStorePassword")))); return SslContextFactory.newInstanceFromTrustStore(keyStore).getServerSSLContext(); } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/test/RemoteServerTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/test/RemoteServerTestCase.java deleted file mode 100644 index 67b3edacc51..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/test/RemoteServerTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.test; - -import org.testng.annotations.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.concurrent.TimeUnit; - -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertTrue; -import static org.testng.AssertJUnit.fail; - -/** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> - */ -public class RemoteServerTestCase { - - @Test - public void requireThatRequestUriFactoryWorks() throws IOException { - RemoteServer server = RemoteServer.newInstance(); - try { - server.newRequestUri((String)null); - fail(); - } catch (NullPointerException e) { - - } - try { - server.newRequestUri((URI)null); - fail(); - } catch (NullPointerException e) { - - } - try { - server.newRequestUri("foo"); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getCause() instanceof URISyntaxException); - } - URI requestUri = server.newRequestUri("/foo?baz=cox#bar"); - URI serverUri = server.connectionSpec(); - assertEquals(serverUri.getScheme(), requestUri.getScheme()); - assertEquals(serverUri.getUserInfo(), requestUri.getUserInfo()); - assertEquals(serverUri.getHost(), requestUri.getHost()); - assertEquals(serverUri.getPort(), requestUri.getPort()); - assertEquals("/foo", requestUri.getPath()); - assertEquals("baz=cox", requestUri.getQuery()); - assertEquals("bar", requestUri.getFragment()); - assertTrue(server.close(60, TimeUnit.SECONDS)); - } -} diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index 53309ef3dd8..3eeeb7adb66 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_define_module( fastos vespalog vespalib + staging_vespalib config_cloudconfig APPS diff --git a/logd/src/apps/logd/main.cpp b/logd/src/apps/logd/main.cpp index 00654060107..70df928dfd2 100644 --- a/logd/src/apps/logd/main.cpp +++ b/logd/src/apps/logd/main.cpp @@ -5,6 +5,7 @@ #include <logd/forward.h> #include <logd/conf.h> #include <logd/watch.h> +#include <logd/state.h> #include <vespa/config/common/exceptions.h> #include <csignal> #include <unistd.h> @@ -29,9 +30,8 @@ int main(int, char**) const char *cfid = getenv("VESPA_CONFIG_ID"); try { - std::unique_ptr<ConfSub> subscriberP; - subscriberP.reset(new ConfSub(fwd, config::ConfigUri(cfid))); - ConfSub & subscriber(*subscriberP); + ConfSub subscriber(fwd, config::ConfigUri(cfid)); + StateReporter stateReporter; int sleepcount = 0; while (true) { @@ -39,6 +39,8 @@ int main(int, char**) try { subscriber.latch(); + stateReporter.setStatePort(subscriber.getStatePort()); + stateReporter.gotConf(subscriber.generation()); int fd = subscriber.getservfd(); if (fd >= 0) { sleepcount = 0 ; // connection OK, reset sleep time diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index b436ef52876..3ef48f9255a 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_library(logd STATIC service.cpp cmdbuf.cpp perform.cpp + state.cpp DEPENDS ) vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) diff --git a/logd/src/logd/conf.cpp b/logd/src/logd/conf.cpp index 8a53317c286..716ba97effd 100644 --- a/logd/src/logd/conf.cpp +++ b/logd/src/logd/conf.cpp @@ -42,6 +42,8 @@ ConfSub::configure(std::unique_ptr<LogdConfig> cfg) _use_logserver = newconf.logserver.use; _newConf = true; } + _statePort = newconf.stateport; + ForwardMap forwardMap; forwardMap[Logger::fatal] = newconf.loglevel.fatal.forward; forwardMap[Logger::error] = newconf.loglevel.error.forward; @@ -145,6 +147,7 @@ ConfSub::closeConn() ConfSub::ConfSub(Forwarder &fw, const config::ConfigUri & configUri) : _logPort(0), _logserverfd(-1), + _statePort(0), _rotate_size(INT_MAX), _rotate_age(INT_MAX), _remove_meg(INT_MAX), diff --git a/logd/src/logd/conf.h b/logd/src/logd/conf.h index ce135d153cd..7a91630ff93 100644 --- a/logd/src/logd/conf.h +++ b/logd/src/logd/conf.h @@ -10,6 +10,7 @@ private: char _logServer[256]; int _logPort; int _logserverfd; + int _statePort; int _rotate_size; int _rotate_age; int _remove_meg; @@ -31,6 +32,7 @@ public: ConfSub(Forwarder &fw, const config::ConfigUri & configUri); ~ConfSub(); + int getStatePort() const { return _statePort; } int getservfd() const { return _logserverfd; } int getRotateSize() const { return _rotate_size; } int getRotateAge() const { return _rotate_age; } @@ -39,6 +41,7 @@ public: bool useLogserver() const { return _use_logserver; } void configure(std::unique_ptr<cloud::config::log::LogdConfig> cfg); + size_t generation() const { return _subscriber.getGeneration(); } }; } // namespace diff --git a/logd/src/logd/state.cpp b/logd/src/logd/state.cpp new file mode 100644 index 00000000000..b73183d5987 --- /dev/null +++ b/logd/src/logd/state.cpp @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> +LOG_SETUP(""); + +#include "state.h" + +namespace logdemon { + +StateReporter::StateReporter() +{ +} + +void +StateReporter::setStatePort(int statePort) +{ + if (statePort != _port) { + _port = statePort; + _server.reset(new vespalib::StateServer(_port, _health, _metrics, _components)); + LOG(info, "state server listening on port %d", _server->getListenPort()); + } +} + +void +StateReporter::gotConf(size_t generation) +{ + vespalib::ComponentConfigProducer::Config conf("logd", generation); + _components.addConfig(conf); +} + +} // namespace diff --git a/logd/src/logd/state.h b/logd/src/logd/state.h new file mode 100644 index 00000000000..7abcc7eb93c --- /dev/null +++ b/logd/src/logd/state.h @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/net/simple_health_producer.h> +#include <vespa/vespalib/net/simple_metrics_producer.h> +#include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/net/generic_state_handler.h> + +namespace logdemon { + +class StateReporter { + int _port; + std::unique_ptr<vespalib::StateServer> _server; + vespalib::SimpleHealthProducer _health; + vespalib::SimpleMetricsProducer _metrics; + vespalib::SimpleComponentConfigProducer _components; +public: + StateReporter(); + void setStatePort(int statePort); + void gotConf(size_t generation); +}; + +} // namespace diff --git a/logd/src/main/resources/configdefinitions/logd.def b/logd/src/main/resources/configdefinitions/logd.def index 96c6bf5a616..7f7e69d3260 100644 --- a/logd/src/main/resources/configdefinitions/logd.def +++ b/logd/src/main/resources/configdefinitions/logd.def @@ -1,6 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=cloud.config.log +## Port to serve status and metrics on +stateport int default=0 + ## Host to contact the logserver on. logserver.host string default="localhost" diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp index 04f2e87c812..bb35aa4d03e 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp +++ b/memfilepersistence/src/tests/spi/memfiletestutils.cpp @@ -7,14 +7,15 @@ #include <vespa/memfilepersistence/memfile/memfilecache.h> #include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/persistence/spi/test.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <sys/time.h> using document::DocumentType; +using document::test::makeBucketSpace; using storage::spi::test::makeSpiBucket; -using storage::spi::test::makeBucketSpace; namespace storage { namespace memfile { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index f6fd8c3bd18..d3af55cdff5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.DockerImpl; @@ -101,6 +102,7 @@ public class DockerOperationsImpl implements DockerOperations { String configServers = String.join(",", environment.getConfigServerHosts()); Docker.CreateContainerCommand command = docker.createContainerCommand( nodeSpec.wantedDockerImage.get(), + ContainerResources.from(nodeSpec.minCpuCores, nodeSpec.minMainMemoryAvailableGb), containerName, nodeSpec.hostname) .withManagedBy(MANAGER_NAME) @@ -122,14 +124,11 @@ public class DockerOperationsImpl implements DockerOperations { // TODO: Enforce disk constraints long minMainMemoryAvailableMb = (long) (nodeSpec.minMainMemoryAvailableGb * 1024); if (minMainMemoryAvailableMb > 0) { - command.withMemoryInMb(minMainMemoryAvailableMb); // VESPA_TOTAL_MEMORY_MB is used to make any jdisc container think the machine // only has this much physical memory (overrides total memory reported by `free -m`). command.withEnvironment("VESPA_TOTAL_MEMORY_MB", Long.toString(minMainMemoryAvailableMb)); } - command.withCpuShares((int) Math.round(10 * nodeSpec.minCpuCores)); - logger.info("Starting new container with args: " + command); command.create(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index ddb04f1249d..2d22f4c4ccf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -147,7 +147,8 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void stopNodeAgentServices(List<String> hostnames) { - nodeAgents.values().stream() + // Each container may spend 1-1:30 minutes stopping + nodeAgents.values().parallelStream() .filter(nodeAgent -> hostnames.contains(nodeAgent.getHostname())) .forEach(NodeAgent::stopServices); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index e5e19ff69e4..906e6b55640 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -232,6 +232,7 @@ public class NodeAdminStateUpdater { nodesToSuspend.addAll(nodesInActiveState); nodesToSuspend.add(dockerHostHostName); orchestrator.suspend(dockerHostHostName, nodesToSuspend); + log.info("Orchestrator allows suspension of " + nodesToSuspend); // The node agent services are stopped by this thread, which is OK only // because the node agents are frozen (see above). diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 453012e9791..09eb14039e8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerException; import com.yahoo.vespa.hosted.dockerapi.DockerExecTimeoutException; @@ -318,6 +319,13 @@ public class NodeAgentImpl implements NodeAgent { if (!existingContainer.state.isRunning()) { return Optional.of("Container no longer running"); } + + ContainerResources wantedContainerResources = ContainerResources.from( + nodeSpec.minCpuCores, nodeSpec.minMainMemoryAvailableGb); + if (!wantedContainerResources.equals(existingContainer.resources)) { + return Optional.of("Container should be running with different resource allocation, wanted: " + + wantedContainerResources + ", actual: " + existingContainer.resources); + } return Optional.empty(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 43170c49a33..616a18d2f2f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -81,7 +81,7 @@ public class DockerOperationsImplTest { } private Container makeContainer(String name, Container.State state, int pid) { - final Container container = new Container(name + ".fqdn", new DockerImage("mock"), + final Container container = new Container(name + ".fqdn", new DockerImage("mock"), null, new ContainerName(name), state, pid); when(docker.getContainer(eq(container.name))).thenReturn(Optional.of(container)); return container; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index dc111251af7..a1cc1850d23 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; @@ -34,13 +35,14 @@ public class DockerMock implements Docker { @Override public CreateContainerCommand createContainerCommand( DockerImage dockerImage, + ContainerResources containerResources, ContainerName containerName, String hostName) { synchronized (monitor) { callOrderVerifier.add("createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName); containersByContainerName.put( - containerName, new Container(hostName, dockerImage, containerName, Container.State.RUNNING, 2)); + containerName, new Container(hostName, dockerImage, containerResources, containerName, Container.State.RUNNING, 2)); } return new StartContainerCommandMock(); @@ -90,7 +92,7 @@ public class DockerMock implements Docker { callOrderVerifier.add("stopContainer with " + containerName); Container container = containersByContainerName.get(containerName); containersByContainerName.put(containerName, - new Container(container.hostname, container.image, container.name, Container.State.EXITED, 0)); + new Container(container.hostname, container.image, container.resources, container.name, Container.State.EXITED, 0)); } } @@ -174,16 +176,6 @@ public class DockerMock implements Docker { } @Override - public CreateContainerCommand withMemoryInMb(long megaBytes) { - return this; - } - - @Override - public CreateContainerCommand withCpuShares(int shares) { - return this; - } - - @Override public CreateContainerCommand withNetworkMode(String mode) { return this; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index 9ce48dac55b..a699377b4c3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -141,7 +141,7 @@ public class AclMaintainerTest { private Container makeContainer(String hostname, Container.State state, int pid) { final ContainerName containerName = new ContainerName(hostname); - final Container container = new Container(hostname, new DockerImage("mock"), + final Container container = new Container(hostname, new DockerImage("mock"), null, containerName, state, pid); containers.add(container); return container; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index ceaa1d58f92..cce000f858a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -6,6 +6,7 @@ import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStatsImpl; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; @@ -229,6 +230,44 @@ public class NodeAgentImplTest { } @Test + public void containerIsRestartedIfFlavorChanged() throws Exception { + final long wantedRestartGeneration = 1; + final long currentRestartGeneration = 1; + ContainerNodeSpec.Builder specBuilder = nodeSpecBuilder + .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) + .nodeState(Node.State.active) + .wantedVespaVersion(vespaVersion) + .vespaVersion(vespaVersion) + .wantedRestartGeneration(wantedRestartGeneration) + .currentRestartGeneration(currentRestartGeneration); + + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + ContainerNodeSpec firstSpec = specBuilder.build(); + ContainerNodeSpec secondSpec = specBuilder.minDiskAvailableGb(200).build(); + ContainerNodeSpec thirdSpec = specBuilder.minCpuCores(4).build(); + + when(nodeRepository.getContainerNodeSpec(hostName)) + .thenReturn(Optional.of(firstSpec)) + .thenReturn(Optional.of(secondSpec)) + .thenReturn(Optional.of(thirdSpec)); + when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); + when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(201326592000L)); + + nodeAgent.converge(); + nodeAgent.converge(); + nodeAgent.converge(); + + InOrder inOrder = inOrder(orchestrator, dockerOperations); + inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator).resume(any(String.class)); + inOrder.verify(orchestrator).suspend(any(String.class)); + inOrder.verify(dockerOperations).removeContainer(any()); + inOrder.verify(dockerOperations).startContainer(eq(containerName), eq(thirdSpec)); + inOrder.verify(orchestrator).resume(any(String.class)); + } + + @Test public void noRestartIfOrchestratorSuspendFails() throws Exception { final long wantedRestartGeneration = 2; final long currentRestartGeneration = 1; @@ -611,6 +650,7 @@ public class NodeAgentImplTest { Optional.of(new Container( hostName, dockerImage, + ContainerResources.from(MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB), containerName, isRunning ? Container.State.RUNNING : Container.State.EXITED, isRunning ? 1 : 0)) : diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 1c81d97ddea..266d91e7e3e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.NodeType; import com.yahoo.transaction.Mutex; @@ -24,12 +23,12 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Maintains information in the node repo about when this node last responded to ping @@ -76,21 +75,15 @@ public class NodeFailer extends Maintainer { @Override protected void maintain() { // Ready nodes - updateNodeLivenessEventsForReadyNodes(); - for (Node node : readyNodesWhichAreDead()) { - // Docker hosts and nodes do not run Vespa services - if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER || node.type() == NodeType.host) continue; - if ( ! throttle(node)) nodeRepository().fail(node.hostname(), - Agent.system, "Not receiving config requests from node"); - } - - for (Node node : readyNodesWithHardwareFailure()) - if ( ! throttle(node)) nodeRepository().fail(node.hostname(), - Agent.system, "Node has hardware failure"); + try (Mutex lock = nodeRepository().lockUnallocated()) { + updateNodeLivenessEventsForReadyNodes(); - for (Node node: readyNodesWithHardwareDivergence()) - if ( ! throttle(node)) nodeRepository().fail(node.hostname(), - Agent.system, "Node hardware diverges from spec"); + getReadyNodesByFailureReason().forEach((node, reason) -> { + if (!throttle(node)) { + nodeRepository().fail(node.hostname(), Agent.system, reason); + } + }); + } // Active nodes for (Node node : determineActiveNodeDownStatus()) { @@ -103,59 +96,55 @@ public class NodeFailer extends Maintainer { private void updateNodeLivenessEventsForReadyNodes() { // Update node last request events through ZooKeeper to collect request to all config servers. // We do this here ("lazily") to avoid writing to zk for each config request. - try (Mutex lock = nodeRepository().lockUnallocated()) { - for (Node node : nodeRepository().getNodes(Node.State.ready)) { - Optional<Instant> lastLocalRequest = hostLivenessTracker.lastRequestFrom(node.hostname()); - if ( ! lastLocalRequest.isPresent()) continue; - - Optional<History.Event> recordedRequest = node.history().event(History.Event.Type.requested); - if ( ! recordedRequest.isPresent() || recordedRequest.get().at().isBefore(lastLocalRequest.get())) { - History updatedHistory = node.history().with(new History.Event(History.Event.Type.requested, - Agent.system, - lastLocalRequest.get())); - nodeRepository().write(node.with(updatedHistory)); - } + for (Node node : nodeRepository().getNodes(Node.State.ready)) { + Optional<Instant> lastLocalRequest = hostLivenessTracker.lastRequestFrom(node.hostname()); + if ( ! lastLocalRequest.isPresent()) continue; + + Optional<History.Event> recordedRequest = node.history().event(History.Event.Type.requested); + if ( ! recordedRequest.isPresent() || recordedRequest.get().at().isBefore(lastLocalRequest.get())) { + History updatedHistory = node.history().with(new History.Event(History.Event.Type.requested, + Agent.system, + lastLocalRequest.get())); + nodeRepository().write(node.with(updatedHistory)); } } } - private List<Node> readyNodesWhichAreDead() { - // Allow requests some time to be registered in case all config servers have been down - if (constructionTime.isAfter(clock.instant().minus(nodeRequestInterval).minus(nodeRequestInterval) )) - return Collections.emptyList(); - - // Nodes are taken as dead if they have not made a config request since this instant. - // Add 10 minutes to the down time limit to allow nodes to make a request that infrequently. - Instant oldestAcceptableRequestTime = clock.instant().minus(downTimeLimit).minus(nodeRequestInterval); - - return nodeRepository().getNodes(Node.State.ready).stream() - .filter(node -> wasMadeReadyBefore(oldestAcceptableRequestTime, node)) - .filter(node -> ! hasRecordedRequestAfter(oldestAcceptableRequestTime, node)) - .collect(Collectors.toList()); + private Map<Node, String> getReadyNodesByFailureReason() { + Instant oldestAcceptableRequestTime = + // Allow requests some time to be registered in case all config servers have been down + constructionTime.isAfter(clock.instant().minus(nodeRequestInterval.multipliedBy(2))) ? + Instant.EPOCH : + + // Nodes are taken as dead if they have not made a config request since this instant. + // Add 10 minutes to the down time limit to allow nodes to make a request that infrequently. + clock.instant().minus(downTimeLimit).minus(nodeRequestInterval); + + Map<Node, String> nodesByFailureReason = new HashMap<>(); + for (Node node : nodeRepository().getNodes(Node.State.ready)) { + if (! hasNodeRequestedConfigAfter(node, oldestAcceptableRequestTime)) { + nodesByFailureReason.put(node, "Not receiving config requests from node"); + } else if (node.status().hardwareFailureDescription().isPresent()) { + nodesByFailureReason.put(node, "Node has hardware failure"); + } else if (node.status().hardwareDivergence().isPresent()) { + nodesByFailureReason.put(node, "Node has hardware divergence"); + } + } + return nodesByFailureReason; } - private List<Node> readyNodesWithHardwareDivergence() { - return nodeRepository().getNodes(Node.State.ready).stream() - .filter(node -> node.status().hardwareDivergence().isPresent()) - .collect(Collectors.toList()); + private boolean hasNodeRequestedConfigAfter(Node node, Instant instant) { + return !wasMadeReadyBefore(node, instant) || hasRecordedRequestAfter(node, instant); } - private boolean wasMadeReadyBefore(Instant instant, Node node) { + private boolean wasMadeReadyBefore(Node node, Instant instant) { Optional<History.Event> readiedEvent = node.history().event(History.Event.Type.readied); - if ( ! readiedEvent.isPresent()) return false; - return readiedEvent.get().at().isBefore(instant); + return readiedEvent.map(event -> event.at().isBefore(instant)).orElse(false); } - private boolean hasRecordedRequestAfter(Instant instant, Node node) { + private boolean hasRecordedRequestAfter(Node node, Instant instant) { Optional<History.Event> lastRequest = node.history().event(History.Event.Type.requested); - if ( ! lastRequest.isPresent()) return false; - return lastRequest.get().at().isAfter(instant); - } - - private List<Node> readyNodesWithHardwareFailure() { - return nodeRepository().getNodes(Node.State.ready).stream() - .filter(node -> node.status().hardwareFailureDescription().isPresent()) - .collect(Collectors.toList()); + return lastRequest.map(event -> event.at().isAfter(instant)).orElse(false); } private boolean applicationSuspended(Node node) { @@ -272,18 +261,18 @@ public class NodeFailer extends Maintainer { private boolean throttle(Node node) { if (throttlePolicy == ThrottlePolicy.disabled) return false; Instant startOfThrottleWindow = clock.instant().minus(throttlePolicy.throttleWindow); - List<Node> nodes = nodeRepository().getNodes().stream() - // Do not consider Docker containers when throttling - .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - .collect(Collectors.toList()); + List<Node> nodes = nodeRepository().getNodes(); long recentlyFailedNodes = nodes.stream() .map(n -> n.history().event(History.Event.Type.failed)) .filter(Optional::isPresent) .map(Optional::get) .filter(failedEvent -> failedEvent.at().isAfter(startOfThrottleWindow)) .count(); - boolean throttle = recentlyFailedNodes >= Math.max(nodes.size() * throttlePolicy.fractionAllowedToFail, - throttlePolicy.minimumAllowedToFail); + int allowedFailedNodes = (int) Math.max(nodes.size() * throttlePolicy.fractionAllowedToFail, + throttlePolicy.minimumAllowedToFail); + + boolean throttle = allowedFailedNodes < recentlyFailedNodes || + (allowedFailedNodes == recentlyFailedNodes && node.type() != NodeType.host); if (throttle) { log.info(String.format("Want to fail node %s, but throttling is in effect: %s", node.hostname(), throttlePolicy.toHumanReadableString())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index bc50de719aa..5ffa99e0de1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -97,10 +97,10 @@ class Activator { List<Node> updated = new ArrayList<>(); for (Node node : nodes) { HostSpec hostSpec = getHost(node.hostname(), hosts); - node = hostSpec.membership().get().retired() - ? node.retire(clock.instant()) - : node.unretire(); + node = hostSpec.membership().get().retired() ? node.retire(clock.instant()) : node.unretire(); node = node.with(node.allocation().get().with(hostSpec.membership().get())); + if (hostSpec.flavor().isPresent()) // Docker nodes may change flavor + node = node.with(hostSpec.flavor().get()); updated.add(node); } return updated; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 0bf48b805aa..fae178adb87 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -49,7 +49,7 @@ class GroupPreparer { List<Node> surplusActiveNodes, MutableInteger highestIndex, int nofSpares, BiConsumer<List<Node>, String> debugRecorder) { try (Mutex lock = nodeRepository.lock(application)) { - // Use new, ready nodes. Lock ready pool to ensure that nodes are not grabbed by others. + // Lock ready pool to ensure that ready nodes are not simultaneously grabbed by others try (Mutex readyLock = nodeRepository.lockUnallocated()) { // Create a prioritized set of nodes diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index b9f682bc79f..d9d7b4a5d12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -90,15 +90,15 @@ class NodeAllocation { // conditions on which we want to retire nodes that were allocated previously if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) wantToRetireNode = true; - if ( !hasCompatibleFlavor(offered)) wantToRetireNode = true; + if ( ! hasCompatibleFlavor(offered)) wantToRetireNode = true; if ( offered.flavor().isRetired()) wantToRetireNode = true; if ( offered.status().wantToRetire()) wantToRetireNode = true; - if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) { + if (( ! saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) { accepted.add(acceptNode(offeredPriority, wantToRetireNode)); } } - else if (! saturated() && hasCompatibleFlavor(offered)) { + else if ( ! saturated() && hasCompatibleFlavor(offered)) { if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) { ++rejectedWithClashingParentHost; continue; @@ -233,6 +233,12 @@ class NodeAllocation { } } } + + // Update flavor of allocated docker nodes as we can change it in place + for (PrioritizableNode node : nodes) { + if (node.node.allocation().isPresent()) + node.node = requestedNodes.assignRequestedFlavor(node.node); + } return nodes.stream().map(n -> n.node).collect(Collectors.toList()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 10d562eff98..4c6f1b022a4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Flavor; +import com.yahoo.vespa.hosted.provision.Node; import java.util.Objects; @@ -39,6 +40,13 @@ public interface NodeSpec { /** Returns a specification of a fraction of all the nodes of this. It is assumed the argument is a valid divisor. */ NodeSpec fraction(int divisor); + /** + * Assigns the flavor requested by this to the given node and returns it, + * if one is requested and it is allowed to change. + * Otherwise, the node is returned unchanged. + */ + Node assignRequestedFlavor(Node node); + static NodeSpec from(int nodeCount, Flavor flavor) { return new CountNodeSpec(nodeCount, flavor); } @@ -51,31 +59,36 @@ public interface NodeSpec { class CountNodeSpec implements NodeSpec { private final int count; - private final Flavor flavor; + private final Flavor requestedFlavor; public CountNodeSpec(int count, Flavor flavor) { Objects.requireNonNull(flavor, "A flavor must be specified"); this.count = count; - this.flavor = flavor; + this.requestedFlavor = flavor; } + // TODO: Remove usage of this public Flavor getFlavor() { - return flavor; + return requestedFlavor; } + // TODO: Remove usage of this public int getCount() { return count; } @Override public NodeType type() { return NodeType.tenant; } @Override - public boolean isCompatible(Flavor flavor) { return flavor.satisfies(this.flavor); } + public boolean isCompatible(Flavor flavor) { + if (flavor.satisfies(requestedFlavor)) return true; + return requestedFlavorCanBeAchievedByResizing(flavor); + } @Override - public boolean matchesExactly(Flavor flavor) { return flavor.equals(this.flavor); } + public boolean matchesExactly(Flavor flavor) { return flavor.equals(this.requestedFlavor); } @Override - public boolean specifiesNonStockFlavor() { return ! flavor.isStock(); } + public boolean specifiesNonStockFlavor() { return ! requestedFlavor.isStock(); } @Override public boolean fulfilledBy(int count) { return count >= this.count; } @@ -87,10 +100,23 @@ public interface NodeSpec { public int surplusGiven(int count) { return count - this.count; } @Override - public NodeSpec fraction(int divisor) { return new CountNodeSpec(count/divisor, flavor); } + public NodeSpec fraction(int divisor) { return new CountNodeSpec(count/divisor, requestedFlavor); } + + @Override + public Node assignRequestedFlavor(Node node) { + // Docker nodes can change flavor in place + if (requestedFlavorCanBeAchievedByResizing(node.flavor())) + return node.with(requestedFlavor); + return node; + } @Override - public String toString() { return "request for " + count + " nodes of " + flavor; } + public String toString() { return "request for " + count + " nodes of " + requestedFlavor; } + + /** Docker nodes can be downsized in place */ + private boolean requestedFlavorCanBeAchievedByResizing(Flavor flavor) { + return flavor.isDocker() && requestedFlavor.isDocker() && flavor.isLargerThan(requestedFlavor); + } } @@ -128,6 +154,9 @@ public interface NodeSpec { public NodeSpec fraction(int divisor) { return this; } @Override + public Node assignRequestedFlavor(Node node) { return node; } + + @Override public String toString() { return "request for all nodes of type '" + type + "'"; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index b16ce5f818e..797453b12c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; +import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; import java.io.IOException; @@ -48,15 +49,18 @@ import static com.yahoo.vespa.config.SlimeUtils.optionalString; */ public class NodesApiHandler extends LoggingRequestHandler { + private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final NodeRepositoryMaintenance maintenance; private final NodeFlavors nodeFlavors; private static final String nodeTypeKey = "type"; - public NodesApiHandler(Executor executor, AccessLog accessLog, NodeRepository nodeRepository, + public NodesApiHandler(Executor executor, AccessLog accessLog, Orchestrator orchestrator, + NodeRepository nodeRepository, NodeRepositoryMaintenance maintenance, NodeFlavors flavors) { super(executor, accessLog); + this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; this.maintenance = maintenance; this.nodeFlavors = flavors; @@ -89,10 +93,10 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); if (path.equals( "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command", "maintenance"); - if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, nodeRepository); - if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, nodeRepository); - if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, nodeRepository); - if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, nodeRepository); + if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); + if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); + if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); + if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 78bd2b5e165..7a0ac908731 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; @@ -13,6 +14,9 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; +import com.yahoo.vespa.orchestrator.HostNameNotFoundException; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.status.HostStatus; import java.io.IOException; import java.io.OutputStream; @@ -36,16 +40,19 @@ class NodesResponse extends HttpResponse { private final NodeFilter filter; private final boolean recursive; + private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final Slime slime; - public NodesResponse(ResponseType responseType, HttpRequest request, NodeRepository nodeRepository) { + public NodesResponse(ResponseType responseType, HttpRequest request, + Orchestrator orchestrator, NodeRepository nodeRepository) { super(200); this.parentUrl = toParentUrl(request); this.nodeParentUrl = toNodeParentUrl(request); filter = NodesApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); + this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; slime = new Slime(); @@ -154,6 +161,11 @@ class NodesResponse extends HttpResponse { object.setLong("currentRestartGeneration", node.allocation().get().restartGeneration().current()); object.setString("wantedDockerImage", nodeRepository.dockerImage().withTag(node.allocation().get().membership().cluster().vespaVersion()).asString()); object.setString("wantedVespaVersion", node.allocation().get().membership().cluster().vespaVersion().toFullString()); + try { + object.setBool("allowedToBeDown", + orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN); + } + catch (HostNameNotFoundException e) {/* ok */ } } object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index bcc18cfd876..7de138fa954 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -45,8 +45,12 @@ public class MockNodeRepository extends NodeRepository { */ public MockNodeRepository(MockCurator curator, NodeFlavors flavors) throws Exception { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), - new MockNameResolver().mockAnyLookup(), new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); + new MockNameResolver() + .addRecord("test-container-1", "::2") + .mockAnyLookup(), + new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); this.flavors = flavors; + curator.setConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); populate(); } @@ -133,4 +137,4 @@ public class MockNodeRepository extends NodeRepository { provisioner.activate(transaction, application, hosts); transaction.commit(); } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 6bcb9426373..0e0195a5bed 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -185,8 +185,8 @@ public class NodeFailTester { } public void allNodesMakeAConfigRequestExcept(List<Node> deadNodes) { - for (Node node : nodeRepository.getNodes(NodeType.tenant)) { - if ( ! deadNodes.contains(node) && node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + for (Node node : nodeRepository.getNodes()) { + if ( ! deadNodes.contains(node)) hostLivenessTracker.receivedRequestFrom(node.hostname()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index c78663415ef..b9b871dfd1f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import org.junit.Test; @@ -66,7 +65,7 @@ public class NodeFailerTest { assertEquals( 4, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); } - // Failures are detected on two ready nodes, which are then failed + // Hardware failures are detected on two ready nodes, which are then failed Node readyFail1 = tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).get(2); Node readyFail2 = tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).get(3); tester.nodeRepository.write(readyFail1.with(readyFail1.status().withHardwareFailureDescription(Optional.of("memory_mcelog")))); @@ -166,18 +165,16 @@ public class NodeFailerTest { tester.createReadyNodes(1, 16, "docker"); // For a day all nodes work so nothing happens - for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) { - tester.clock.advance(Duration.ofMinutes(5)); + for (int minutes = 0, interval = 30; minutes < 24 * 60; minutes += interval) { + tester.clock.advance(Duration.ofMinutes(interval)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals( 5, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); } List<Node> ready = tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready); - List<Node> readyHosts = tester.nodeRepository.getNodes(NodeType.host, Node.State.ready); - // Two ready nodes die and a ready docker node "dies" - // (Vespa does not run when in ready state for docker node, so it does not make config requests) + // Two ready nodes and a ready docker node die, but only 2 of those are failed out tester.clock.advance(Duration.ofMinutes(180)); Node dockerNode = ready.stream().filter(node -> node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER).findFirst().get(); List<Node> otherNodes = ready.stream() @@ -188,18 +185,13 @@ public class NodeFailerTest { assertEquals( 3, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); assertEquals( 2, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); - // Another ready node die + // Another ready node dies and the node that died earlier, are allowed to fail tester.clock.advance(Duration.ofDays(1)); tester.allNodesMakeAConfigRequestExcept(otherNodes.get(0), otherNodes.get(2), dockerNode, otherNodes.get(3)); tester.failer.run(); - assertEquals( 2, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); - assertEquals(ready.get(1), tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).get(0)); - assertEquals( 3, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); - - // Ready Docker hosts do not make config requests - tester.allNodesMakeAConfigRequestExcept(readyHosts.get(0), readyHosts.get(1), readyHosts.get(2)); - tester.failer.run(); - assertEquals(3, tester.nodeRepository.getNodes(NodeType.host, Node.State.ready).size()); + assertEquals( 1, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); + assertEquals(otherNodes.get(1), tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready).get(0)); + assertEquals( 4, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.failed).size()); } @Test @@ -207,8 +199,8 @@ public class NodeFailerTest { NodeFailTester tester = NodeFailTester.withTwoApplicationsOnDocker(7); // For a day all nodes work so nothing happens - for (int minutes = 0; minutes < 24 * 60; minutes += 5 ) { - tester.clock.advance(Duration.ofMinutes(5)); + for (int minutes = 0, interval = 30; minutes < 24 * 60; minutes += interval) { + tester.clock.advance(Duration.ofMinutes(interval)); tester.allNodesMakeAConfigRequestExcept(); tester.failer.run(); assertEquals(8, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); @@ -247,10 +239,10 @@ public class NodeFailerTest { Node downTenant1 = tester.nodeRepository.getNodes(NodeType.tenant, Node.State.active).get(0); tester.serviceMonitor.setHostDown(downTenant1.hostname()); - // nothing happens the first 45 minutes - for (int minutes = 0; minutes < 45; minutes += 5 ) { + // nothing happens during the entire day because of the failure throttling + for (int minutes = 0, interval = 30; minutes < 24 * 60; minutes += interval) { tester.failer.run(); - tester.clock.advance(Duration.ofMinutes(5)); + tester.clock.advance(Duration.ofMinutes(interval)); tester.allNodesMakeAConfigRequestExcept(); assertEquals(3 + 1, tester.nodeRepository.getNodes(Node.State.failed).size()); } @@ -374,14 +366,12 @@ public class NodeFailerTest { public void node_failing_throttle() { // Throttles based on a absolute number in small zone { - NodeFailTester tester = NodeFailTester.withNoApplications(); - List<Node> readyNodes = tester.createReadyNodes(50); - List<Node> readyDockerNodes = tester.createReadyNodes(50, 50, "docker"); + // 50 regular tenant nodes, 10 hosts with each 3 tenant nodes, total 90 nodes + NodeFailTester tester = NodeFailTester.withTwoApplicationsOnDocker(10); + List<Node> readyNodes = tester.createReadyNodes(50, 30); + List<Node> hosts = tester.nodeRepository.getNodes(NodeType.host); List<Node> deadNodes = readyNodes.subList(0, 4); - // Fail 10 Docker containers, should not impact throttling policy - readyDockerNodes.subList(0, 10) - .forEach(node -> tester.nodeRepository.fail(node.hostname(), Agent.system, "Failed in test")); // 2 hours pass, 4 nodes die for (int minutes = 0, interval = 30; minutes < 2 * 60; minutes += interval) { @@ -391,7 +381,7 @@ public class NodeFailerTest { // 2 nodes are failed (the minimum amount that are always allowed to fail) tester.failer.run(); - assertEquals(2, getNonDockerFailedNodes(tester.nodeRepository).size()); + assertEquals(2, tester.nodeRepository.getNodes(Node.State.failed).size()); // 6 more hours pass, no more nodes are failed for (int minutes = 0, interval = 30; minutes < 6 * 60; minutes += interval) { @@ -399,15 +389,39 @@ public class NodeFailerTest { tester.allNodesMakeAConfigRequestExcept(deadNodes); } tester.failer.run(); - assertEquals(2, getNonDockerFailedNodes(tester.nodeRepository).size()); + assertEquals(2, tester.nodeRepository.getNodes(Node.State.failed).size()); - // 18 more hours pass, it's now 24 hours since the first 2 failed. The remaining 2 are failed - for (int minutes = 0, interval = 30; minutes < 18 * 60; minutes += interval) { + // 2 docker hosts now fail, 1 of them (with all its children is allowed to fail) + hosts.subList(0, 2).forEach(host -> { + tester.serviceMonitor.setHostDown(host.hostname()); + deadNodes.add(host); + }); + tester.failer.run(); + tester.clock.advance(Duration.ofMinutes(61)); + tester.allNodesMakeAConfigRequestExcept(deadNodes); + + tester.failer.run(); + assertEquals(6, tester.nodeRepository.getNodes(Node.State.failed).size()); + + // 24 more hours pass without any other nodes being failed out + for (int minutes = 0, interval = 30; minutes <= 23 * 60; minutes += interval) { tester.clock.advance(Duration.ofMinutes(interval)); tester.allNodesMakeAConfigRequestExcept(deadNodes); } tester.failer.run(); - assertEquals(4, getNonDockerFailedNodes(tester.nodeRepository).size()); + assertEquals(6, tester.nodeRepository.getNodes(Node.State.failed).size()); + + // Next, the 2 ready nodes that were dead from the start are failed out, and finally + // the second host and all its children are failed + tester.clock.advance(Duration.ofMinutes(30)); + tester.failer.run(); + assertEquals(12, tester.nodeRepository.getNodes(Node.State.failed).size()); + + // Nothing else to fail + tester.clock.advance(Duration.ofHours(25)); + tester.allNodesMakeAConfigRequestExcept(deadNodes); + tester.failer.run(); + assertEquals(12, tester.nodeRepository.getNodes(Node.State.failed).size()); } // Throttles based on percentage in large zone @@ -443,12 +457,6 @@ public class NodeFailerTest { } } - /** Get all failed nodes that are not Docker containers */ - private static List<Node> getNonDockerFailedNodes(NodeRepository nodeRepository) { - return nodeRepository.getNodes(Node.State.failed).stream() - .filter(node -> node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - .collect(Collectors.toList()); - } /** * Selects the first parent host that: diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 134808b7114..c38d6f6bd40 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -178,8 +178,8 @@ public class AclProvisioningTest { // Populate repo List<Node> dockerHostNodes = tester.makeReadyNodes(2, "default", NodeType.host); Node dockerHostNodeUnderTest = dockerHostNodes.get(0); - List<Node> dockerNodes = tester.makeReadyDockerNodes(5, "docker1", - dockerHostNodeUnderTest.hostname()); + List<Node> dockerNodes = tester.makeReadyDockerNodes(5, "dockerSmall", + dockerHostNodeUnderTest.hostname()); List<NodeAcl> acls = tester.nodeRepository().getNodeAcls(dockerHostNodeUnderTest, true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index d17aa8ef741..a6760ef37ce 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertEquals; */ public class DockerProvisioningTest { - private static final String dockerFlavor = "docker1"; + private static final String dockerFlavor = "dockerSmall"; @Test public void docker_application_deployment() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index dea65f432ec..afdce0d25cc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -232,6 +232,29 @@ public class ProvisioningTest { } @Test + public void application_deployment_with_inplace_downsize() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); + + ApplicationId application1 = tester.makeApplicationId(); + + tester.makeReadyNodes(14, "dockerLarge"); + + // deploy + SystemState state1 = prepare(application1, 2, 2, 4, 4, "dockerLarge", tester); + tester.activate(application1, state1.allHosts); + + // redeploy with smaller docker flavor - causes in-place flavor change + SystemState state2 = prepare(application1, 2, 2, 4, 4, "dockerSmall", tester); + tester.activate(application1, state2.allHosts); + + assertEquals(12, tester.getNodes(application1, Node.State.active).asList().size()); + for (Node node : tester.getNodes(application1, Node.State.active).asList()) + assertEquals("Node changed flavor in place", "dockerSmall", node.flavor().name()); + assertEquals("No nodes are retired", + 0, tester.getNodes(application1, Node.State.active).retired().size()); + } + + @Test public void application_deployment_multiple_flavors_default_per_type() { ConfigserverConfig.Builder config = new ConfigserverConfig.Builder(); config.environment("prod"); @@ -506,7 +529,7 @@ public class ProvisioningTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Unknown flavor 'nonexisting'. Flavors are [default, docker1, large, old-large1, old-large2, small, v-4-8-100]", e.getMessage()); + assertEquals("Unknown flavor 'nonexisting'. Flavors are [default, dockerLarge, dockerSmall, large, old-large1, old-large2, small, v-4-8-100]", e.getMessage()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index d9ce04513a5..3c932f492db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -94,7 +94,8 @@ public class ProvisioningTester implements AutoCloseable { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2); - b.addFlavor("docker1", 1., 1., 10, Flavor.Type.DOCKER_CONTAINER).cost(1); + b.addFlavor("dockerSmall", 1., 1., 10, Flavor.Type.DOCKER_CONTAINER).cost(1); + b.addFlavor("dockerLarge", 2., 1., 20, Flavor.Type.DOCKER_CONTAINER).cost(3); b.addFlavor("v-4-8-100", 4., 8., 100, Flavor.Type.VIRTUAL_MACHINE).cost(4); b.addFlavor("old-large1", 2., 4., 100, Flavor.Type.BARE_METAL).cost(6); b.addFlavor("old-large2", 2., 5., 100, Flavor.Type.BARE_METAL).cost(14); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 37b2f54da4d..54e0c59ff00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -40,9 +39,20 @@ public class RestApiTest { private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; + private JDisc container; + + @Before + public void startContainer() { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + } + + @After + public void stopContainer() { + container.close(); + } + /** This test gives examples of all the requests that can be made to nodes/v2 */ @Test - @Ignore /** TODO re-enable this and verify correctness */ public void test_requests() throws Exception { // GET assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); @@ -54,28 +64,28 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); // GET with filters - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host2.yahoo.com%20host1.yahoo.com"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "active-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=parent1.yahoo.com,parent.host.yahoo.com"), "parent-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); // POST restart command assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host2.yahoo.com", new byte[0], Request.Method.POST)); assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertRestart(4, new Request("http://localhost:8080/nodes/v2/command/restart", + assertRestart(9, new Request("http://localhost:8080/nodes/v2/command/restart", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"restartGeneration\":3"); // POST reboot command - assertReboot(5, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", + assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", new byte[0], Request.Method.POST)); assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot", + assertReboot(15, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"rebootGeneration\":4"); @@ -106,9 +116,9 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // DELETE a provisioned node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", new byte[0], Request.Method.DELETE), - "{\"message\":\"Removed host11.yahoo.com\"}"); + "{\"message\":\"Removed host9.yahoo.com\"}"); // PUT nodes ready assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host8.yahoo.com", @@ -125,15 +135,15 @@ public class RestApiTest { "{\"message\":\"Moved host8.yahoo.com to ready\"}"); // PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host3.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host3.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"), + "{\"message\":\"Moved host2.yahoo.com to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"state\":\"failed\""); // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host3.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host3.yahoo.com to active\"}"); + "{\"message\":\"Moved host2.yahoo.com to active\"}"); // PUT a node in parked ... assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", @@ -147,30 +157,29 @@ public class RestApiTest { "{\"message\":\"Removed host8.yahoo.com\"}"); // or, PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), + "{\"message\":\"Moved test-container-1 to failed\"}"); + assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-container-1"), "\"state\":\"failed\""); // ... and deallocate it such that it moves to dirty and is recycled - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to dirty\"}"); + "{\"message\":\"Moved test-container-1 to dirty\"}"); // ... and set it back to ready as if this was from the node-admin with the temporary state rest api - assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/test-container-1", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host6.yahoo.com to ready\"}"); + "{\"message\":\"Marked following nodes as available for new allocation: test-container-1\"}"); // Put a host in failed and make sure it's children are also failed - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/parent1.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host10.yahoo.com, host5.yahoo.com, parent1.yahoo.com to failed\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), + "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed"), "{\"nodes\":[" + - "{\"url\":\"http://localhost:8080/nodes/v2/node/parent1.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}," + "{\"url\":\"http://localhost:8080/nodes/v2/node/host5.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/host10.yahoo.com\"}]}"); - + "{\"url\":\"http://localhost:8080/nodes/v2/node/host4.yahoo.com\"}]}"); // Update (PATCH) a node (multiple fields can also be sent in one request body) assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", @@ -514,13 +523,6 @@ public class RestApiTest { } } - private JDisc container; - @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); } - @After - public void stopContainer() { container.close(); } - private String asDockerNodeJson(String hostname, String parentHostname, int additionalIpCount, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + createIpAddresses(ipAddress) + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json index d1df5b83f24..c67ba904f9a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json @@ -1,8 +1,13 @@ { "nodes": [ + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json), @include(node6.json), - @include(node3.json), @include(node2.json), - @include(node1.json) + @include(docker-container1.json), + @include(node4.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json index 1d4d97315cd..4581ecba73d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json @@ -1,6 +1,6 @@ { "nodes": [ - @include(node2.json), - @include(node1.json) + @include(node6.json), + @include(node2.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json new file mode 100644 index 00000000000..dae92aae091 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json @@ -0,0 +1,5 @@ +{ + "nodes": [ + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json new file mode 100644 index 00000000000..47a2c012b17 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json @@ -0,0 +1,8 @@ +{ + "nodes": [ + @include(node6.json), + @include(node2.json), + @include(docker-container1.json), + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json new file mode 100644 index 00000000000..7823ed0431d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json @@ -0,0 +1,56 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/test-container-1", + "id": "test-container-1", + "state": "active", + "type": "tenant", + "hostname": "test-container-1", + "parentHostname": "dockerhost3.yahoo.com", + "openStackId": "fake-test-container-1", + "flavor": "docker", + "canonicalFlavor": "docker", + "minDiskAvailableGb": 100.0, + "minMainMemoryAvailableGb": 0.5, + "description": "Flavor-name-is-docker", + "minCpuCores": 0.2, + "fastDisk": true, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "reserved", + "at": 123, + "agent": "system" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::2" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json new file mode 100644 index 00000000000..a13dfae927f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json new file mode 100644 index 00000000000..f7a1d6ab9a9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + "id": "dockerhost2.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost2.yahoo.com", + "openStackId": "dockerhost2", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json new file mode 100644 index 00000000000..f877d33672f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com", + "id": "dockerhost3.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost3.yahoo.com", + "openStackId": "dockerhost3", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 2, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json new file mode 100644 index 00000000000..913cf9852aa --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com", + "id": "dockerhost4.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost4.yahoo.com", + "openStackId": "dockerhost4", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 3, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json new file mode 100644 index 00000000000..685b0a52b15 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json @@ -0,0 +1,70 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com", + "id": "dockerhost5.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost5.yahoo.com", + "openStackId": "dockerhost5", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 4, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::2", + "::3", + "::4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index fea4fb8d4d2..c09829e7f85 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -28,6 +28,9 @@ "name":"OperatorChangeApplicationMaintainer" }, { + "name":"ProvisionedExpirer" + }, + { "name":"RetiredEarlyExpirer" }, { @@ -43,4 +46,4 @@ "inactive":[ "NodeFailer" ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index 56523f58164..08e4904e35c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index cb24df66cf8..65e2b1435dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -30,6 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, "vespaVersion": "5.104.142", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 30057dda1d7..aa60cfae815 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index 6106e25e75e..cb250e2033b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -1,7 +1,7 @@ { "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", "id": "host4.yahoo.com", - "state": "reserved", + "state": "failed", "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "parent.yahoo.com", @@ -12,35 +12,36 @@ "minMainMemoryAvailableGb": 12.0, "description": "Flavor-name-is-medium-disk", "minCpuCores": 6.0, - "fastDisk":true, + "fastDisk": true, "environment": "BARE_METAL", "owner": { - "tenant": "tenant1", - "application": "application1", - "instance": "instance1" + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" }, "membership": { - "clustertype": "container", - "clusterid": "id1", + "clustertype": "content", + "clusterid": "id3", "group": "0", - "index": 1, + "index": 0, "retired": false }, - "restartGeneration": 0, + "restartGeneration": 1, "currentRestartGeneration": 1, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "rebootGeneration": 2, + "allowedToBeDown": false, + "rebootGeneration": 3, "currentRebootGeneration": 1, "vespaVersion": "6.43.0", "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.43.0", "hostedVersion": "6.43.0", "convergedStateVersion": "6.43.0", - "failCount": 0, + "failCount": 1, "hardwareFailure": true, "hardwareFailureDescription": "memory_mcelog", - "wantToRetire" : true, - "wantToDeprovision" : true, + "wantToRetire": true, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -50,7 +51,7 @@ { "event": "readied", "at": 123, - "agent": "system" + "agent": "system" }, { "event": "reserved", @@ -58,11 +59,24 @@ "agent": "application" }, { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "failed", + "at": 123, + "agent": "operator" + }, + { "event": "rebooted", "at": 123, "agent": "system" } ], - "ipAddresses":["127.0.0.1", "::1"], - "additionalIpAddresses":[] + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index 1e9138283f7..1b705a0ee8c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -30,6 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, "vespaVersion": "6.41.0", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json index be9f3d78663..a5043746749 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index bd3b15f16ba..c5382dfd1a7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json index a9feed81674..475b914989b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json @@ -1,14 +1,19 @@ { "nodes": [ @include(node7.json), - @include(parent1.json), + @include(node3.json), @include(node10.json), - @include(node4.json), + @include(node1.json), + @include(docker-node4.json), @include(node6.json), - @include(node3.json), + @include(docker-node5.json), + @include(docker-node2.json), @include(node2.json), - @include(node1.json), + @include(docker-node1.json), + @include(docker-node3.json), + @include(docker-container1.json), + @include(node4.json), @include(node55.json), @include(node5.json) ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json index 67b65259f8a..3bfaa95d5ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json @@ -4,31 +4,46 @@ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/parent1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host10.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host4.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host6.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host3.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host2.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host1.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host55.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" }, { - "url":"http://localhost:8080/nodes/v2/node/host5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/test-container-1" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" } ] -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json deleted file mode 100644 index 81ca0465c4b..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "nodes": [ - @include(node10.json), - @include(node5.json) - ] -}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json index 4ee1d5ed9b9..183f81ee3e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json @@ -9,23 +9,28 @@ "ready": { "url": "http://localhost:8080/nodes/v2/state/ready", "nodes": [ - @include(parent1.json) + @include(node3.json) ] }, "reserved": { "url": "http://localhost:8080/nodes/v2/state/reserved", "nodes": [ @include(node10.json), - @include(node4.json) + @include(node1.json) ] }, "active": { "url": "http://localhost:8080/nodes/v2/state/active", "nodes": [ + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json), @include(node6.json), - @include(node3.json), @include(node2.json), - @include(node1.json) + @include(docker-container1.json), + @include(node4.json) ] }, "inactive": { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java index e0f5b0ecf4f..c7aae6ea93d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java @@ -20,6 +20,7 @@ public class RetryingClusterControllerClientFactory implements ClusterController // TODO: Figure this port out dynamically. public static final int HARDCODED_CLUSTERCONTROLLER_PORT = 19050; public static final String CLUSTERCONTROLLER_API_PATH = "/"; + public static final String CLUSTERCONTROLLER_SCHEME = "http"; private static final int CLUSTER_CONTROLLER_CONNECT_TIMEOUT_MS = 1000; private static final int CLUSTER_CONTROLLER_READ_TIMEOUT_MS = 1000; @@ -39,7 +40,7 @@ public class RetryingClusterControllerClientFactory implements ClusterController String clusterName) { Set<HostName> clusterControllerSet = clusterControllers.stream().collect(Collectors.toSet()); JaxRsStrategy<ClusterControllerJaxRsApi> jaxRsApi - = new JaxRsStrategyFactory(clusterControllerSet, HARDCODED_CLUSTERCONTROLLER_PORT, jaxRsClientFactory) + = new JaxRsStrategyFactory(clusterControllerSet, HARDCODED_CLUSTERCONTROLLER_PORT, jaxRsClientFactory, CLUSTERCONTROLLER_SCHEME) .apiWithRetries(ClusterControllerJaxRsApi.class, CLUSTERCONTROLLER_API_PATH); return new ClusterControllerClientImpl(jaxRsApi, clusterName); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactory.java index ca757adfea3..7459f0a6b11 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactory.java @@ -16,6 +16,7 @@ import java.util.logging.Logger; public class SingleInstanceClusterControllerClientFactory implements ClusterControllerClientFactory { public static final int CLUSTERCONTROLLER_HARDCODED_PORT = 19050; + public static final String CLUSTERCONTROLLER_HARDCODED_SCHEME = "http"; public static final String CLUSTERCONTROLLER_API_PATH = "/"; private static final Logger log = Logger.getLogger(SingleInstanceClusterControllerClientFactory.class.getName()); @@ -44,7 +45,8 @@ public class SingleInstanceClusterControllerClientFactory implements ClusterCont port, jaxRsClientFactory, ClusterControllerJaxRsApi.class, - CLUSTERCONTROLLER_API_PATH); + CLUSTERCONTROLLER_API_PATH, + CLUSTERCONTROLLER_HARDCODED_SCHEME); return new ClusterControllerClientImpl(strategy, clusterName); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index edfd35e0573..52994db2b88 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -68,13 +68,11 @@ public class HostResource implements HostApi { serviceInstance.serviceStatus().name())) .collect(Collectors.toList()); - // TODO: Use applicationUri.toString() and hostServices once nobody is using <6.164 - // TODO: Enable testing again in HostResourceTest.getHost_works return new GetHostResponse( host.getHostName().s(), host.getHostStatus().name(), - null, - null); + applicationUri.toString(), + hostServices); } catch (HostNameNotFoundException e) { log.log(LogLevel.INFO, "Host not found: " + hostName); throw new NotFoundException(e); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java index aaddb01197c..b26d11a113f 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java @@ -43,6 +43,7 @@ public class SingleInstanceClusterControllerClientFactoryTest { eq(ClusterControllerJaxRsApi.class), any(HostName.class), anyInt(), + anyString(), anyString())) .thenReturn(mockApi); } @@ -70,7 +71,8 @@ public class SingleInstanceClusterControllerClientFactoryTest { ClusterControllerJaxRsApi.class, HOST_NAME_1, PORT, - PATH); + PATH, + "http"); } @Test @@ -99,7 +101,8 @@ public class SingleInstanceClusterControllerClientFactoryTest { equalTo(HOST_NAME_2), equalTo(HOST_NAME_3)))), eq(PORT), - eq(PATH)); + eq(PATH), + eq("http")); } private static ConfigId clusterControllerConfigId(final int index) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java index 95fad5e56ed..8595c8a31a4 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -28,7 +28,6 @@ import static org.junit.Assert.assertEquals; * * @author smorgrav */ -@Ignore("Fails with JVM crashing with OoM on CentOS") public class ApplicationSuspensionResourceTest { static final String BASE_PATH = "/orchestrator/v1/suspensions/applications"; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index ca0596d8926..65309440aee 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -48,7 +48,6 @@ import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; import static org.fest.assertions.Assertions.assertThat; import static org.fest.assertions.Fail.fail; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; @@ -358,17 +357,13 @@ public class HostResourceTest { Collections.singletonList(serviceInstance)); when(orchestrator.getHost(hostName)).thenReturn(host); GetHostResponse response = hostResource.getHost(hostName.s()); + assertEquals("https://foo.com/bar", response.applicationUrl()); assertEquals("hostname", response.hostname()); assertEquals("ALLOWED_TO_BE_DOWN", response.state()); - - // See TODO in HostResource: - // assertEquals("https://foo.com/bar", response.applicationUrl()); - // assertEquals(1, response.services().size()); - // assertEquals("clusterId", response.services().get(0).clusterId); - // assertEquals("configId", response.services().get(0).configId); - // assertEquals("UP", response.services().get(0).serviceStatus); - // assertEquals("serviceType", response.services().get(0).serviceType); - assertNull(response.applicationUrl()); - assertNull(response.services()); + assertEquals(1, response.services().size()); + assertEquals("clusterId", response.services().get(0).clusterId); + assertEquals("configId", response.services().get(0).configId); + assertEquals("UP", response.services().get(0).serviceStatus); + assertEquals("serviceType", response.services().get(0).serviceType); } } diff --git a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp index 30e6e2236de..c9e6b44508e 100644 --- a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp +++ b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp @@ -4,6 +4,7 @@ #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/documentid.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> @@ -11,6 +12,7 @@ using namespace storage::spi; using namespace storage; +using document::test::makeBucketSpace; using dummy::BucketContent; namespace { @@ -79,7 +81,7 @@ TEST_F("require that setClusterState sets the cluster state", Fixture) { document::DocumentTypeRepo::SP repo; dummy::DummyPersistence provider(repo); - provider.setClusterState(state); + provider.setClusterState(makeBucketSpace(), state); EXPECT_EQUAL(false, provider.getClusterState().nodeUp()); } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 30aa19c6a86..885d3e9aad7 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -7,6 +7,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/state/state.h> #include <vespa/vdslib/state/node.h> #include <vespa/vdslib/state/nodestate.h> @@ -18,8 +19,8 @@ using document::BucketId; using document::BucketSpace; +using document::test::makeBucketSpace; using storage::spi::test::makeSpiBucket; -using storage::spi::test::makeBucketSpace; namespace storage::spi { @@ -2136,7 +2137,7 @@ void ConformanceTest::testBucketActivation() Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); - spi->setClusterState(createClusterState()); + spi->setClusterState(makeBucketSpace(), createClusterState()); spi->createBucket(bucket, context); CPPUNIT_ASSERT(!spi->getBucketInfo(bucket).getBucketInfo().isActive()); @@ -2155,9 +2156,9 @@ void ConformanceTest::testBucketActivation() CPPUNIT_ASSERT(spi->getBucketInfo(bucket).getBucketInfo().isActive()); // Setting node down should clear active flag. - spi->setClusterState(createClusterState(lib::State::DOWN)); + spi->setClusterState(makeBucketSpace(), createClusterState(lib::State::DOWN)); CPPUNIT_ASSERT(!spi->getBucketInfo(bucket).getBucketInfo().isActive()); - spi->setClusterState(createClusterState(lib::State::UP)); + spi->setClusterState(makeBucketSpace(), createClusterState(lib::State::UP)); CPPUNIT_ASSERT(!spi->getBucketInfo(bucket).getBucketInfo().isActive()); // Actively clearing it should of course also clear it @@ -2184,7 +2185,7 @@ void ConformanceTest::testBucketActivationSplitAndJoin() Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, 1); Document::SP doc2 = testDocMan.createRandomDocumentAtLocation(0x06, 2); - spi->setClusterState(createClusterState()); + spi->setClusterState(makeBucketSpace(), createClusterState()); spi->createBucket(bucketC, context); spi->put(bucketC, Timestamp(1), doc1, context); spi->put(bucketC, Timestamp(2), doc2, context); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 5c38679f52c..9a4f094d32c 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -370,7 +370,7 @@ DummyPersistence::getModifiedBuckets(BucketSpace) const } Result -DummyPersistence::setClusterState(const ClusterState& c) +DummyPersistence::setClusterState(BucketSpace, const ClusterState& c) { vespalib::MonitorGuard lock(_monitor); _clusterState.reset(new ClusterState(c)); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 50a4562ea3b..f89afbcac5d 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -142,7 +142,7 @@ public: */ BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; - Result setClusterState(const ClusterState& newState) override; + Result setClusterState(BucketSpace bucketSpace, const ClusterState& newState) override; Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; BucketInfoResult getBucketInfo(const Bucket&) const override; Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 2460259cfe7..07482873467 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -55,7 +55,7 @@ public: /** * Default impl empty. */ - Result setClusterState(const ClusterState&) override { return Result(); } + Result setClusterState(BucketSpace, const ClusterState&) override { return Result(); } /** * Default impl empty. diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp index e338c76fb88..76b0a3c4686 100644 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp @@ -113,10 +113,10 @@ Impl::listBuckets(BucketSpace bucketSpace, PartitionId v1) const } Result -Impl::setClusterState(const ClusterState& v1) +Impl::setClusterState(BucketSpace bucketSpace, const ClusterState& v1) { PRE_PROCESS(3); - Result r(_next->setClusterState(v1)); + Result r(_next->setClusterState(bucketSpace, v1)); POST_PROCESS(3, r); return r; } diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h index 8ec2e2dd1bc..e169ad098c7 100644 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h @@ -35,7 +35,7 @@ public: Result initialize() override; PartitionStateListResult getPartitionStates() const override; BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - Result setClusterState(const ClusterState&) override; + Result setClusterState(BucketSpace bucketSpace, const ClusterState&) override; Result setActiveState(const Bucket&, BucketInfo::ActiveState) override; BucketInfoResult getBucketInfo(const Bucket&) const override; Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index b10ed618e88..d7a5fa2b4a4 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -93,7 +93,7 @@ struct PersistenceProvider * supplied (changes that relate to the distributor will not cause an * update here). */ - virtual Result setClusterState(const ClusterState&) = 0; + virtual Result setClusterState(BucketSpace bucketSpace, const ClusterState&) = 0; /** * Sets the bucket state to active or inactive. After this returns, diff --git a/persistence/src/vespa/persistence/spi/test.cpp b/persistence/src/vespa/persistence/spi/test.cpp index e376bd6b4d4..b58d527c598 100644 --- a/persistence/src/vespa/persistence/spi/test.cpp +++ b/persistence/src/vespa/persistence/spi/test.cpp @@ -1,33 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "test.h" +#include <vespa/document/test/make_bucket_space.h> using document::BucketId; using document::BucketSpace; +using document::test::makeBucketSpace; namespace storage::spi::test { -BucketSpace makeBucketSpace() -{ - return BucketSpace::placeHolder(); -} - -BucketSpace makeBucketSpace(const vespalib::string &docTypeName) -{ - // Used by persistence conformance test to map fron document type name - // to bucket space. See document::TestDocRepo for known document types. - if (docTypeName == "no") { - return BucketSpace(2); - } else if (docTypeName == "testdoctype2") { - return BucketSpace(1); - } else { - return makeBucketSpace(); - } -} - Bucket makeSpiBucket(BucketId bucketId, PartitionId partitionId) { - return Bucket(document::Bucket(BucketSpace::placeHolder(), bucketId), partitionId); + return Bucket(document::Bucket(makeBucketSpace(), bucketId), partitionId); } Bucket makeSpiBucket(BucketId bucketId) diff --git a/persistence/src/vespa/persistence/spi/test.h b/persistence/src/vespa/persistence/spi/test.h index 06bade014ab..445cfc91213 100644 --- a/persistence/src/vespa/persistence/spi/test.h +++ b/persistence/src/vespa/persistence/spi/test.h @@ -8,8 +8,6 @@ namespace storage::spi::test { // Helper functions used by unit tests -document::BucketSpace makeBucketSpace(); -document::BucketSpace makeBucketSpace(const vespalib::string &docTypeName); Bucket makeSpiBucket(document::BucketId bucketId, PartitionId partitionId); Bucket makeSpiBucket(document::BucketId bucketId); @@ -39,7 +39,7 @@ </snapshots> <id>bintray-yahoo-maven</id> <name>bintray</name> - <url>http://yahoo.bintray.com/maven</url> + <url>https://yahoo.bintray.com/maven</url> </repository> </repositories> @@ -544,6 +544,12 @@ <version>${commons-lang.version}</version> </dependency> <dependency> + <!-- This version is exported by jdisc via jcl-over-slf4j. --> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + </dependency> + <dependency> <groupId>commons-net</groupId> <artifactId>commons-net</artifactId> <version>2.0</version> diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp index 4ee9a0635b0..33ad4bc5024 100644 --- a/searchcore/src/apps/proton/downpersistence.cpp +++ b/searchcore/src/apps/proton/downpersistence.cpp @@ -45,7 +45,7 @@ DownPersistence::listBuckets(BucketSpace, PartitionId) const } Result -DownPersistence:: setClusterState(const ClusterState&) +DownPersistence:: setClusterState(BucketSpace, const ClusterState&) { return Result(); } diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h index 9e64b89f065..0a602c4467e 100644 --- a/searchcore/src/apps/proton/downpersistence.h +++ b/searchcore/src/apps/proton/downpersistence.h @@ -29,7 +29,7 @@ public: Result initialize() override; PartitionStateListResult getPartitionStates() const override; BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - Result setClusterState(const ClusterState&) override; + Result setClusterState(BucketSpace, const ClusterState&) override; Result setActiveState(const Bucket&, BucketInfo::ActiveState) override; BucketInfoResult getBucketInfo(const Bucket&) const override; Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 0ea3b41f232..8bb1911e595 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -9,7 +9,7 @@ #include <vespa/document/base/testdocman.h> #include <vespa/fastos/file.h> #include <vespa/persistence/conformancetest/conformancetest.h> -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/matching/querylimiter.h> @@ -34,7 +34,6 @@ LOG_SETUP("persistenceconformance_test"); using namespace config; using namespace proton; using namespace cloud::config::filedistribution; -using namespace storage::spi::test; using namespace vespa::config::search::core; using namespace vespa::config::search::summary; using namespace vespa::config::search; @@ -45,6 +44,7 @@ using document::DocumentType; using document::DocumentTypeRepo; using document::DocumenttypesConfig; using document::TestDocMan; +using document::test::makeBucketSpace; using search::TuneFileDocumentDB; using search::index::DummyFileHeaderContext; using search::index::Schema; diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 0347da0c52c..f60863ef0b0 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -36,12 +36,9 @@ using vespa::config::search::core::RankingConstantsConfig; using vespalib::eval::ConstantValue; using vespalib::eval::ErrorValue; using vespalib::eval::TensorSpec; -using vespalib::eval::TensorValue; using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; - -using ErrorConstant = vespalib::eval::SimpleConstantValue<ErrorValue>; -using TensorConstant = vespalib::eval::SimpleConstantValue<TensorValue>; +using vespalib::eval::SimpleConstantValue; class App : public FastOS_Application { @@ -66,11 +63,11 @@ struct DummyConstantValueRepo : IConstantValueRepo { for (const auto &entry: cfg.constant) { if (entry.name == name) { const auto &engine = DefaultTensorEngine::ref(); - auto tensor = engine.create(TensorSpec(entry.type)); - return std::make_unique<TensorConstant>(engine.type_of(*tensor), std::move(tensor)); + auto tensor = engine.from_spec(TensorSpec(entry.type)); + return std::make_unique<SimpleConstantValue>(std::move(tensor)); } } - return std::make_unique<ErrorConstant>(ValueType::error_type()); + return std::make_unique<SimpleConstantValue>(std::make_unique<ErrorValue>()); } }; diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index ce0205af29f..cb1df2df0c5 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -41,6 +41,7 @@ LOG_SETUP("attribute_test"); #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcommon/attribute/iattributevector.h> namespace vespa { namespace config { namespace search {}}} @@ -55,6 +56,7 @@ using proton::ImportedAttributesRepo; using proton::test::AttributeUtils; using search::TuneFileAttributes; using search::attribute::BitVectorSearchCache; +using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; using search::attribute::ReferenceAttribute; using search::index::DummyFileHeaderContext; @@ -69,10 +71,11 @@ using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; -typedef search::attribute::Config AVConfig; -typedef search::attribute::BasicType AVBasicType; -typedef search::attribute::CollectionType AVCollectionType; -typedef SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > Int32AttributeVector; +using AVConfig = search::attribute::Config; +using AVBasicType = search::attribute::BasicType; +using AVCollectionType = search::attribute::CollectionType; +using Int32AttributeVector = SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> >; +using LidVector = LidVectorContext::LidVector; namespace { @@ -156,6 +159,9 @@ struct Fixture void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) { _aw->remove(serialNum, lid, immediateCommit, emptyCallback); } + void remove(const LidVector &lidVector, SerialNum serialNum, bool immediateCommit = true) { + _aw->remove(lidVector, serialNum, immediateCommit, emptyCallback); + } void commit(SerialNum serialNum) { _aw->forceCommit(serialNum, emptyCallback); } @@ -293,23 +299,23 @@ TEST_F("require that attribute writer handles predicate put", Fixture) EXPECT_TRUE(it.valid()); } +void +assertUndefined(const IAttributeVector &attr, uint32_t docId) +{ + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(attr.getInt(docId))); +} + TEST_F("require that attribute writer handles remove", Fixture) { AttributeVector::SP a1 = f.addAttribute("a1"); AttributeVector::SP a2 = f.addAttribute("a2"); - Schema s; - s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); - s.addAttributeField(Schema::AttributeField("a2", schema::DataType::INT32, CollectionType::SINGLE)); - - DocBuilder idb(s); - fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); f.remove(2, 0); - EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a1->getInt(0))); - EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(0))); + TEST_DO(assertUndefined(*a1, 0)); + TEST_DO(assertUndefined(*a2, 0)); f.remove(2, 0); // same sync token as previous try { @@ -321,6 +327,24 @@ TEST_F("require that attribute writer handles remove", Fixture) } } +TEST_F("require that attribute writer handles batch remove", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = f.addAttribute("a2"); + fillAttribute(a1, 4, 22, 1); + fillAttribute(a2, 4, 33, 1); + + LidVector lidsToRemove = {1,3}; + f.remove(lidsToRemove, 2); + + TEST_DO(assertUndefined(*a1, 1)); + EXPECT_EQUAL(22, a1->getInt(2)); + TEST_DO(assertUndefined(*a1, 3)); + TEST_DO(assertUndefined(*a2, 1)); + EXPECT_EQUAL(33, a2->getInt(2)); + TEST_DO(assertUndefined(*a2, 3)); +} + void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) { attribute::ConstCharContent sbuf; diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index bce3fb7267c..6705527dfa9 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/tensor/default_tensor.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> #include <vespa/eval/tensor/tensor_factory.h> -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/docsummary/docsumcontext.h> @@ -43,9 +43,9 @@ using namespace search::engine; using namespace search::index; using namespace search::transactionlog; using namespace search; -using namespace storage::spi::test; using document::DocumenttypesConfig; +using document::test::makeBucketSpace; using search::TuneFileDocumentDB; using search::index::DummyFileHeaderContext; using search::index::schema::CollectionType; diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index a997b3cc3db..c7006be2804 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -2,7 +2,7 @@ #include <vespa/log/log.h> LOG_SETUP("combiningfeedview_test"); -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/server/combiningfeedview.h> #include <vespa/searchcore/proton/test/test.h> @@ -11,10 +11,10 @@ LOG_SETUP("combiningfeedview_test"); using document::DocumentTypeRepo; using document::DocumentUpdate; +using document::test::makeBucketSpace; using search::IDestructorCallback; using search::SerialNum; using storage::spi::Timestamp; -using storage::spi::test::makeBucketSpace; using namespace proton; typedef std::vector<IFeedView::SP> FeedViewVector; diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 8369ec0630d..1b94ba36dad 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -44,6 +44,7 @@ using namespace search; using namespace searchcorespi; using namespace vespalib; +using document::test::makeBucketSpace; using proton::bucketdb::BucketDBHandler; using proton::bucketdb::IBucketDBHandler; using proton::bucketdb::IBucketDBHandlerInitializer; @@ -52,7 +53,6 @@ using search::test::DirectoryHandler; using searchcorespi::IFlushTarget; using searchcorespi::index::IThreadingService; using storage::spi::Timestamp; -using storage::spi::test::makeBucketSpace; using vespa::config::search::core::ProtonConfig; using vespalib::mkdir; diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 4419e982abf..7aa452c5aa3 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -15,7 +15,7 @@ #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vespalib/testkit/testapp.h> using namespace proton; @@ -24,6 +24,7 @@ using document::Document; using document::DocumentId; using document::DocumentTypeRepo; using document::GlobalId; +using document::test::makeBucketSpace; using proton::bucketdb::BucketCreateNotifier; using search::DocumentIdT; using search::DocumentMetaData; @@ -32,7 +33,6 @@ using search::index::DocBuilder; using search::index::Schema; using storage::spi::BucketInfo; using storage::spi::Timestamp; -using storage::spi::test::makeBucketSpace; using vespalib::make_string; using BlockedReason = IBlockableMaintenanceJob::BlockedReason; diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index f7e09eeec3f..58372c59193 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -3,7 +3,7 @@ #include <tests/proton/common/dummydbowner.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/fastos/file.h> -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcore/proton/attribute/flushableattribute.h> #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/common/statusreport.h> @@ -27,12 +27,12 @@ using namespace cloud::config::filedistribution; using namespace proton; -using namespace storage::spi::test; using namespace vespalib::slime; using document::DocumentType; using document::DocumentTypeRepo; using document::DocumenttypesConfig; +using document::test::makeBucketSpace; using search::TuneFileDocumentDB; using search::index::DummyFileHeaderContext; using search::index::Schema; diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index f20ad01bcf6..b84aa1c1c6c 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/persistence/spi/test.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/common/doctypename.h> @@ -43,6 +43,7 @@ using namespace vespalib::slime; using document::BucketId; using document::Document; using document::DocumentId; +using document::test::makeBucketSpace; using fastos::ClockSystem; using fastos::TimeStamp; using proton::bucketdb::BucketCreateNotifier; @@ -54,7 +55,6 @@ using search::IDestructorCallback; using search::SerialNum; using storage::spi::BucketInfo; using storage::spi::Timestamp; -using storage::spi::test::makeBucketSpace; using vespalib::Slime; using vespalib::makeClosure; using vespalib::makeTask; diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 428cbf60996..6bc7e1b5556 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -15,7 +15,7 @@ using SimpleFlushHandler = test::DummyFlushHandler; using FlushCandidatesList = std::vector<FlushTargetCandidates>; using Config = PrepareRestartFlushStrategy::Config; -const Config DEFAULT_CFG(2.0, 4.0); +const Config DEFAULT_CFG(2.0, 0.0, 4.0); struct SimpleFlushTarget : public test::DummyFlushTarget { @@ -107,7 +107,7 @@ public: : _sortedFlushContexts(&sortedFlushContexts), _numCandidates(sortedFlushContexts.size()), _tlsStats(1000, 11, 110), - _cfg(DEFAULT_CFG) + _cfg(2.0, 3.0, 4.0) {} CandidatesBuilder &flushContexts(const FlushContext::List &sortedFlushContexts) { _sortedFlushContexts = &sortedFlushContexts; @@ -140,28 +140,35 @@ struct CandidatesFixture CandidatesFixture() : emptyContexts(), builder(emptyContexts) {} }; +void +assertCosts(double tlsReplayBytesCost, double tlsReplayOperationsCost, double flushTargetsWriteCost, const FlushTargetCandidates &candidates) +{ + EXPECT_EQUAL(tlsReplayBytesCost, candidates.getTlsReplayCost().bytesCost); + EXPECT_EQUAL(tlsReplayOperationsCost, candidates.getTlsReplayCost().operationsCost); + EXPECT_EQUAL(flushTargetsWriteCost, candidates.getFlushTargetsWriteCost()); + EXPECT_EQUAL(tlsReplayBytesCost + tlsReplayOperationsCost + flushTargetsWriteCost, candidates.getTotalCost()); +} + TEST_F("require that tls replay cost is correct for 100% replay", CandidatesFixture) { - EXPECT_EQUAL(2000, f.builder.replayEnd(110).build().getTlsReplayCost()); + TEST_DO(assertCosts(1000 * 2, 100 * 3, 0, f.builder.replayEnd(110).build())); } TEST_F("require that tls replay cost is correct for 75% replay", CandidatesFixture) { FlushContext::List contexts = ContextsBuilder().add("target1", 10, 0).add("target2", 35, 0).build(); - EXPECT_EQUAL(1500, f.builder.flushContexts(contexts).numCandidates(1).replayEnd(110). - build().getTlsReplayCost()); + TEST_DO(assertCosts(750 * 2, 75 * 3, 0, f.builder.flushContexts(contexts).numCandidates(1).replayEnd(110).build())); } TEST_F("require that tls replay cost is correct for 25% replay", CandidatesFixture) { FlushContext::List contexts = ContextsBuilder().add("target1", 10, 0).add("target2", 85, 0).build(); - EXPECT_EQUAL(500, f.builder.flushContexts(contexts).numCandidates(1).replayEnd(110). - build().getTlsReplayCost()); + TEST_DO(assertCosts(250 * 2, 25 * 3, 0, f.builder.flushContexts(contexts).numCandidates(1).replayEnd(110).build())); } TEST_F("require that tls replay cost is correct for zero operations to replay", CandidatesFixture) { - EXPECT_EQUAL(0, f.builder.replayEnd(10).build().getTlsReplayCost()); + TEST_DO(assertCosts(0, 0, 0, f.builder.replayEnd(10).build())); } TEST_F("require that flush cost is correct for zero flush targets", CandidatesFixture) @@ -172,7 +179,7 @@ TEST_F("require that flush cost is correct for zero flush targets", CandidatesFi TEST_F("require that flush cost is sum of flush targets", CandidatesFixture) { FlushContext::List contexts = ContextsBuilder().add("target1", 20, 1000).add("target2", 30, 2000).build(); - EXPECT_EQUAL(12000, f.builder.flushContexts(contexts).build().getFlushTargetsWriteCost()); + TEST_DO(assertCosts(0, 0, 1000 * 4 + 2000 * 4, f.builder.flushContexts(contexts).build())); } @@ -227,7 +234,7 @@ assertFlushContexts(const vespalib::string &expected, const FlushContext::List & * - handler1: serial numbers 10 -> 110, 1000 bytes * - handler2: serial numbers 10 -> 110, 2000 bytes * - * The cost config is: tlsReplayCost=2.0, flushTargetsWriteCost=4.0. + * The cost config is: tlsReplayByteCost=2.0, tlsReplayOperationCost=0.0, flushTargetsWriteCost=4.0. * The cost of replaying the complete TLS is then: * - handler1: 1000*2.0 = 2000 * - handler2: 2000*2.0 = 4000 diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 9e13cf2ff7d..4a195514db1 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -4,6 +4,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/persistence/spi/documentselection.h> #include <vespa/persistence/spi/test.h> #include <vespa/persistence/spi/test.h> @@ -21,6 +22,7 @@ using document::BucketSpace; using document::Document; using document::DocumentId; using document::DocumentType; +using document::test::makeBucketSpace; using search::DocumentMetaData; using storage::spi::Bucket; using storage::spi::BucketChecksum; @@ -42,7 +44,6 @@ using storage::spi::Selection; using storage::spi::Timestamp; using storage::spi::UpdateResult; using storage::spi::test::makeSpiBucket; -using storage::spi::test::makeBucketSpace; using namespace proton; using namespace vespalib; @@ -610,7 +611,7 @@ TEST_F("require that setClusterState() is routed to handlers", SimpleFixture) { ClusterState state(createClusterState()); - f.engine.setClusterState(state); + f.engine.setClusterState(makeBucketSpace(), state); EXPECT_EQUAL(&state, f.hset.handler1.lastCalc); EXPECT_EQUAL(&state, f.hset.handler2.lastCalc); } diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 6c682ea33e9..b9059338f27 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -170,12 +170,12 @@ struct ProtonConfigOwner : public proton::IProtonConfigurer return getConfigured(); } virtual void reconfigure(std::shared_ptr<ProtonConfigSnapshot> cfg) override { - std::unique_lock<std::mutex> guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); _config.set(cfg); _configured = true; } bool getConfigured() const { - std::unique_lock<std::mutex> guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); return _configured; } BootstrapConfig::SP getBootstrapConfig() { diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index eab889ea28c..c6c810ae72f 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -82,17 +82,31 @@ flush.memory.conservative.disklimitfactor double default=0.5 ## watermark indicating when to go back from conservative to normal mode for the flush strategy. flush.memory.conservative.lowwatermarkfactor double default=0.9 -## The cost of doing replay when replaying the transaction log. +## The cost of replaying a byte when replaying the transaction log. ## -## The number of bytes to replay * replaycost gives an estimate of the -## total cost of replaying the transaction log. +## The estimate of the total cost of replaying the transaction log: +## (number of bytes to replay) * replaycost + (number of operations to replay) * replayoperationcost ## ## The prepare for restart flush strategy will choose a set of components to flush ## such that the cost of flushing these + the cost of replaying the transaction log ## is as low as possible. -flush.preparerestart.replaycost double default=4.0 +flush.preparerestart.replaycost double default=2.0 -## The cost of doing writes when flushing components to disk. +## The cost of replaying an operation when replaying the transaction log. +## +## The estimate of the total cost of replaying the transaction log: +## (number of bytes to replay) * replaycost + (number of operations to replay) * replayoperationcost +## +## The default value is chosen based on the following example: +## Assume we can replay 9 MB/s and this corresponds to 24000 ops/s. +## replayoperationcost = (bytes to replay) * replaycost / (operations to replay) = 9 MB * 2.0 / 24000 = 750 +## +## The prepare for restart flush strategy will choose a set of components to flush +## such that the cost of flushing these + the cost of replaying the transaction log +## is as low as possible. +flush.preparerestart.replayoperationcost double default=750.0 + +## The cost of writing a byte when flushing components to disk. ## ## The number of bytes to write (for a set of flushed components) * writecost ## gives an estimate of the total cost of flushing this set of components. diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp index 7685ddcc328..7b060e793f6 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp @@ -165,7 +165,7 @@ void FastS_SyncSearchAdapter::DoneQuery(FastS_ISearch *, FastS_SearchContext) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _queryDone = true; if (_waitQuery) { _cond.notify_one(); @@ -177,7 +177,7 @@ void FastS_SyncSearchAdapter::DoneDocsums(FastS_ISearch *, FastS_SearchContext) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _docsumsDone = true; if (_waitDocsums) { _cond.notify_one(); diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp index 83312a41875..24668db6024 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp @@ -112,7 +112,7 @@ void FastS_EngineBase::SlowQuery(double limit, double secs, bool silent) { { - std::unique_lock<std::mutex> engineGuard(_lock); + std::lock_guard<std::mutex> engineGuard(_lock); _stats._slowQueryCnt++; _stats._slowQuerySecs += secs; } @@ -127,7 +127,7 @@ void FastS_EngineBase::SlowDocsum(double limit, double secs) { { - std::unique_lock<std::mutex> engineGuard(_lock); + std::lock_guard<std::mutex> engineGuard(_lock); _stats._slowDocsumCnt++; _stats._slowDocsumSecs += secs; } @@ -173,7 +173,7 @@ FastS_EngineBase::SampleQueueLens() double queueLen; double activecnt; - std::unique_lock<std::mutex> engineGuard(_lock); + std::lock_guard<std::mutex> engineGuard(_lock); if (_stats._queueLenSampleCnt > 0) queueLen = (double) _stats._queueLenSampleAcc / (double) _stats._queueLenSampleCnt; else @@ -217,7 +217,7 @@ FastS_EngineBase::MarkBad(uint32_t badness) bool worse = false; { - std::unique_lock<std::mutex> engineGuard(_lock); + std::lock_guard<std::mutex> engineGuard(_lock); if (badness > _badness) { _badness = badness; worse = true; diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 0cfbdc8b69a..85599b9e897 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -1077,7 +1077,7 @@ FastS_FNET_Search::Search(uint32_t searchOffset, // allow FNET responses while requests are being sent { - std::unique_lock<std::mutex> searchGuard(_lock); + std::lock_guard<std::mutex> searchGuard(_lock); ++_pendingQueries; // add Elephant query node to avoid early query done ++_queryNodes; // add Elephant query node to avoid early query done _FNET_mode = FNET_QUERY; @@ -1102,7 +1102,7 @@ FastS_FNET_Search::Search(uint32_t searchOffset, // finalize setup and check if query is still in progress bool done; { - std::unique_lock<std::mutex> searchGuard(_lock); + std::lock_guard<std::mutex> searchGuard(_lock); assert(_queryNodes >= _pendingQueries); for (uint32_t i: send_failed) { // conditional revert of state for failed nodes @@ -1398,7 +1398,7 @@ FastS_FNET_Search::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) ConnectDocsumNodes(ignoreRow); bool done; { - std::unique_lock<std::mutex> searchGuard(_lock); + std::lock_guard<std::mutex> searchGuard(_lock); // patch in engine dependent features and send docsum requests diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp index 6fddfae2ab0..4b272a615a6 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp @@ -125,7 +125,7 @@ FastS_NodeManager::CheckTempFail() _checkTempFailScheduled = false; tempfail = false; { - std::unique_lock<std::mutex> mangerGuard(_managerLock); + std::lock_guard<std::mutex> mangerGuard(_managerLock); FastS_DataSetCollection *dsc = PeekDataSetCollection(); for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) { FastS_DataSetBase *ds; @@ -166,7 +166,7 @@ uint32_t FastS_NodeManager::SetPartMap(const PartitionsConfig& partmap, unsigned int waitms) { - std::unique_lock<std::mutex> configGuard(_configLock); + std::lock_guard<std::mutex> configGuard(_configLock); FastS_DataSetCollDesc *configDesc = new FastS_DataSetCollDesc(); if (!configDesc->ReadConfig(partmap)) { LOG(error, "NodeManager::SetPartMap: Failed to load configuration"); @@ -275,7 +275,7 @@ FastS_NodeManager::SetDataSetCollection(FastS_DataSetCollection *dsc) } else { { - std::unique_lock<std::mutex> managerGuard(_managerLock); + std::lock_guard<std::mutex> managerGuard(_managerLock); _gencnt++; gencnt = _gencnt; @@ -304,7 +304,7 @@ FastS_NodeManager::GetDataSetCollection() { FastS_DataSetCollection *ret; - std::unique_lock<std::mutex> managerGuard(_managerLock); + std::lock_guard<std::mutex> managerGuard(_managerLock); ret = _datasetCollection; FastS_assert(ret != NULL); ret->addRef(); @@ -320,8 +320,8 @@ FastS_NodeManager::ShutdownConfig() FastS_DataSetCollection *old_dsc; { - std::unique_lock<std::mutex> configGuard(_configLock); - std::unique_lock<std::mutex> managerGuard(_managerLock); + std::lock_guard<std::mutex> configGuard(_configLock); + std::lock_guard<std::mutex> managerGuard(_managerLock); _shutdown = true; // disallow SetPartMap dsc = _datasetCollection; _datasetCollection = new FastS_DataSetCollection(_appCtx); @@ -347,7 +347,7 @@ FastS_NodeManager::GetTotalPartitions() uint32_t ret; ret = 0; - std::unique_lock<std::mutex> managerGuard(_managerLock); + std::lock_guard<std::mutex> managerGuard(_managerLock); FastS_DataSetCollection *dsc = PeekDataSetCollection(); for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) { FastS_DataSetBase *ds; @@ -429,7 +429,7 @@ FastS_NodeManager::CheckEvents(FastS_TimeKeeper *timeKeeper) FastS_DataSetCollection *tmp; { - std::unique_lock<std::mutex> managerGuard(_managerLock); + std::lock_guard<std::mutex> managerGuard(_managerLock); old_dsc = _oldDSCList; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp index eac994bb339..f775f4443f8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp @@ -56,7 +56,7 @@ AttributeDirectory::getDirName() const { std::shared_ptr<AttributeDiskLayout> diskLayout; { - std::unique_lock<std::mutex> guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); assert(!_diskLayout.expired()); diskLayout = _diskLayout.lock(); } @@ -204,7 +204,7 @@ void AttributeDirectory::detach() { assert(empty()); - std::unique_lock<std::mutex> guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); _diskLayout.reset(); } @@ -238,7 +238,7 @@ AttributeDirectory::tryGetWriter() bool AttributeDirectory::empty() const { - std::unique_lock<std::mutex> guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); return _snapInfo.snapshots().empty(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 906f71fecb4..a6329db2aee 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -19,6 +19,8 @@ using search::attribute::ImportedAttributeVector; namespace proton { +using LidVector = LidVectorContext::LidVector; + AttributeWriter::WriteContext::WriteContext(uint32_t executorId) : _executorId(executorId), _fieldPaths(), @@ -143,7 +145,15 @@ applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum, AttributeVector &attr) { if (attr.getStatus().getLastSyncToken() < serialNum) { - attr.compactLidSpace(wantedLidLimit); + /* + * If the attribute is an empty placeholder attribute due to + * later config changes removing the attribute then it might + * be smaller than expected during transaction log replay. + */ + attr.commit(); + if (wantedLidLimit <= attr.getCommittedDocIdLimit()) { + attr.compactLidSpace(wantedLidLimit); + } attr.commit(serialNum, serialNum); } } @@ -264,13 +274,48 @@ RemoveTask::run() const auto &attributes = _wc.getAttributes(); for (auto &attrp : attributes) { AttributeVector &attr = *attrp; - // Must use <= due to batch remove + // Must use <= due to how move operations are handled if (attr.getStatus().getLastSyncToken() <= _serialNum) { applyRemoveToAttribute(_serialNum, _lid, _immediateCommit, attr, _onWriteDone); } } } +class BatchRemoveTask : public vespalib::Executor::Task +{ +private: + const AttributeWriter::WriteContext &_writeCtx; + const SerialNum _serialNum; + const LidVector _lidsToRemove; + const bool _immediateCommit; + std::remove_reference_t<AttributeWriter::OnWriteDoneType> _onWriteDone; +public: + BatchRemoveTask(const AttributeWriter::WriteContext &writeCtx, + SerialNum serialNum, + const LidVector &lidsToRemove, + bool immediateCommit, + AttributeWriter::OnWriteDoneType onWriteDone) + : _writeCtx(writeCtx), + _serialNum(serialNum), + _lidsToRemove(lidsToRemove), + _immediateCommit(immediateCommit), + _onWriteDone(onWriteDone) + {} + virtual ~BatchRemoveTask() override {} + virtual void run() override { + for (auto attr : _writeCtx.getAttributes()) { + if (attr->getStatus().getLastSyncToken() < _serialNum) { + for (auto lidToRemove : _lidsToRemove) { + applyRemoveToAttribute(_serialNum, lidToRemove, false, *attr, _onWriteDone); + } + if (_immediateCommit) { + attr->commit(_serialNum, _serialNum); + } + } + } + } +}; + class CommitTask : public vespalib::Executor::Task { const AttributeWriter::WriteContext &_wc; @@ -411,8 +456,9 @@ void AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum, bool immediateCommit, OnWriteDoneType onWriteDone) { - for (const auto &lid : lidsToRemove) { - internalRemove(serialNum, lid, immediateCommit, onWriteDone); + for (const auto &writeCtx : _writeContexts) { + auto removeTask = std::make_unique<BatchRemoveTask>(writeCtx, serialNum, lidsToRemove, immediateCommit, onWriteDone); + _attributeFieldWriter.executeTask(writeCtx.getExecutorId(), std::move(removeTask)); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp index 1fcffa92cce..bb2f99d077b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp @@ -59,7 +59,7 @@ AttributeDiskLayout::getAttributeDir(const vespalib::string &name) std::shared_ptr<AttributeDirectory> AttributeDiskLayout::createAttributeDir(const vespalib::string &name) { - std::unique_lock<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_timed_mutex> guard(_mutex); auto itr = _dirs.find(name); if (itr == _dirs.end()) { auto dir = std::make_shared<AttributeDirectory>(shared_from_this(), name); @@ -81,7 +81,7 @@ AttributeDiskLayout::removeAttributeDir(const vespalib::string &name, search::Se writer->invalidateOldSnapshots(serialNum); writer->removeInvalidSnapshots(); if (writer->removeDiskDir()) { - std::unique_lock<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_timed_mutex> guard(_mutex); auto itr = _dirs.find(name); assert(itr != _dirs.end()); assert(dir.get() == itr->second.get()); @@ -89,7 +89,7 @@ AttributeDiskLayout::removeAttributeDir(const vespalib::string &name, search::Se writer->detach(); } } else { - std::unique_lock<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_timed_mutex> guard(_mutex); auto itr = _dirs.find(name); if (itr != _dirs.end()) { assert(dir.get() != itr->second.get()); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp index da54b909759..0051c209ef9 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp @@ -8,6 +8,7 @@ namespace proton { using search::SerialNum; using Config = PrepareRestartFlushStrategy::Config; +using TlsReplayCost = FlushTargetCandidates::TlsReplayCost; namespace { @@ -25,7 +26,7 @@ calculateReplayStartSerial(const FlushContext::List &sortedFlushContexts, return sortedFlushContexts[numCandidates]->getTarget()->getFlushedSerialNum() + 1; } -double +TlsReplayCost calculateTlsReplayCost(const flushengine::TlsStats &tlsStats, const Config &cfg, SerialNum replayStartSerial) @@ -33,13 +34,13 @@ calculateTlsReplayCost(const flushengine::TlsStats &tlsStats, SerialNum replayEndSerial = tlsStats.getLastSerial(); SerialNum numTotalOperations = replayEndSerial - tlsStats.getFirstSerial() + 1; if (numTotalOperations == 0) { - return 0; + return TlsReplayCost(0.0, 0.0); } double numBytesPerOperation = (double)tlsStats.getNumBytes() / (double)numTotalOperations; SerialNum numOperationsToReplay = replayEndSerial + 1 - replayStartSerial; double numBytesToReplay = numBytesPerOperation * numOperationsToReplay; - return numBytesToReplay * cfg.tlsReplayCost; + return TlsReplayCost((numBytesToReplay * cfg.tlsReplayByteCost), (numOperationsToReplay * cfg.tlsReplayOperationCost)); } double diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h index 5498d8c46a8..ea09989de31 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h @@ -16,10 +16,20 @@ namespace flushengine { class TlsStats; } */ class FlushTargetCandidates { +public: + struct TlsReplayCost { + double bytesCost; + double operationsCost; + TlsReplayCost(double bytesCost_, double operationsCost_) + : bytesCost(bytesCost_), + operationsCost(operationsCost_) + {} + double totalCost() const { return bytesCost + operationsCost; } + }; private: const FlushContext::List *_sortedFlushContexts; // NOTE: ownership is handled outside size_t _numCandidates; - double _tlsReplayCost; + TlsReplayCost _tlsReplayCost; double _flushTargetsWriteCost; using Config = PrepareRestartFlushStrategy::Config; @@ -32,9 +42,9 @@ public: const flushengine::TlsStats &tlsStats, const Config &cfg); - double getTlsReplayCost() const { return _tlsReplayCost; } + TlsReplayCost getTlsReplayCost() const { return _tlsReplayCost; } double getFlushTargetsWriteCost() const { return _flushTargetsWriteCost; } - double getTotalCost() const { return getTlsReplayCost() + getFlushTargetsWriteCost(); } + double getTotalCost() const { return getTlsReplayCost().totalCost() + getFlushTargetsWriteCost(); } FlushContext::List getCandidates() const; }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp index e9df78dbf4f..6cfb8cb6c3d 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp @@ -18,9 +18,11 @@ using Config = PrepareRestartFlushStrategy::Config; using FlushContextsMap = std::map<vespalib::string, FlushContext::List>; using FlushTargetCandidatesList = std::vector<FlushTargetCandidates::UP>; -PrepareRestartFlushStrategy::Config::Config(double tlsReplayCost_, +PrepareRestartFlushStrategy::Config::Config(double tlsReplayByteCost_, + double tlsReplayOperationCost_, double flushTargetWriteCost_) - : tlsReplayCost(tlsReplayCost_), + : tlsReplayByteCost(tlsReplayByteCost_), + tlsReplayOperationCost(tlsReplayOperationCost_), flushTargetWriteCost(flushTargetWriteCost_) { } @@ -108,18 +110,22 @@ findBestTargetsToFlush(const FlushContext::List &unsortedFlushContexts, for (size_t numCandidates = 1; numCandidates <= sortedFlushContexts.size(); ++numCandidates) { FlushTargetCandidates nextSet(sortedFlushContexts, numCandidates, tlsStats, cfg); LOG(debug, "findBestTargetsToFlush(): Created candidate set: " - "flushTargets=[%s], tlsReplayCost=%f, flushTargetsWriteCost=%f, totalCost=%f", + "flushTargets=[%s], tlsReplayBytesCost=%f, tlsReplayOperationsCost=%f, flushTargetsWriteCost=%f, totalCost=%f", toString(nextSet.getCandidates()).c_str(), - nextSet.getTlsReplayCost(), nextSet.getFlushTargetsWriteCost(), + nextSet.getTlsReplayCost().bytesCost, + nextSet.getTlsReplayCost().operationsCost, + nextSet.getFlushTargetsWriteCost(), nextSet.getTotalCost()); if (nextSet.getTotalCost() < bestSet.getTotalCost()) { bestSet = nextSet; } } LOG(info, "findBestTargetsToFlush(): Best candidate set: " - "flushTargets=[%s], tlsReplayCost=%f, flushTargetsWriteCost=%f, totalCost=%f", + "flushTargets=[%s], tlsReplayBytesCost=%f, tlsReplayOperationsCost=%f, flushTargetsWriteCost=%f, totalCost=%f", toString(bestSet.getCandidates()).c_str(), - bestSet.getTlsReplayCost(), bestSet.getFlushTargetsWriteCost(), + bestSet.getTlsReplayCost().bytesCost, + bestSet.getTlsReplayCost().operationsCost, + bestSet.getFlushTargetsWriteCost(), bestSet.getTotalCost()); return bestSet.getCandidates(); } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h index 19a5cf45670..df5c5a9c569 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h @@ -21,9 +21,12 @@ class PrepareRestartFlushStrategy : public IFlushStrategy public: struct Config { - double tlsReplayCost; + double tlsReplayByteCost; + double tlsReplayOperationCost; double flushTargetWriteCost; - Config(double tlsReplayCost_, double flushTargetWriteCost_); + Config(double tlsReplayByteCost_, + double tlsReplayOperationCost_, + double flushTargetWriteCost_); }; private: diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 96ad422009c..83466cd51ad 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -62,7 +62,7 @@ MatchEngine::close() { LOG(debug, "Closing search interface."); { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _closed = true; } @@ -74,21 +74,21 @@ ISearchHandler::SP MatchEngine::putSearchHandler(const DocTypeName &docTypeName, const ISearchHandler::SP &searchHandler) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.putHandler(docTypeName, searchHandler); } ISearchHandler::SP MatchEngine::getSearchHandler(const DocTypeName &docTypeName) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandler(docTypeName); } ISearchHandler::SP MatchEngine::removeSearchHandler(const DocTypeName &docTypeName) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.removeHandler(docTypeName); } @@ -121,7 +121,7 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, ISearchHandler::SP searchHandler; vespalib::SimpleThreadBundle::UP threadBundle = _threadBundlePool.obtain(); { // try to find the match handler corresponding to the specified search doc type - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); DocTypeName docTypeName(*req.get()); searchHandler = _handlers.getHandler(docTypeName); } @@ -130,7 +130,7 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, } else { HandlerMap<ISearchHandler>::Snapshot::UP snapshot; { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } if (snapshot->valid()) { diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 3055bbfb814..e27f1789ed3 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -9,6 +9,7 @@ #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/simple_thread_bundle.h> +#include <mutex> namespace proton { @@ -16,7 +17,7 @@ class MatchEngine : public search::engine::SearchServer, public vespalib::StateExplorer { private: - vespalib::Lock _lock; + std::mutex _lock; const uint32_t _distributionKey; bool _closed; HandlerMap<ISearchHandler> _handlers; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp index 753c84cd9b6..6d05ce8c57d 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp @@ -20,14 +20,14 @@ JobTracker::sampleLoad(time_point now, const std::lock_guard<std::mutex> &guard) void JobTracker::start() { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _sampler.startJob(std::chrono::steady_clock::now()); } void JobTracker::end() { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _sampler.endJob(std::chrono::steady_clock::now()); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 6cdec1ec0f9..c7b01d209ee 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -20,7 +20,6 @@ using storage::spi::PartitionState; using storage::spi::PartitionStateList; using storage::spi::Result; using vespalib::IllegalStateException; -using vespalib::LockGuard; using vespalib::Sequence; using vespalib::make_string; @@ -30,7 +29,7 @@ namespace { class ResultHandlerBase { protected: - vespalib::Lock _lock; + std::mutex _lock; vespalib::CountDownLatch _latch; public: ResultHandlerBase(uint32_t waitCnt); @@ -55,7 +54,7 @@ public: ~GenericResultHandler(); void handle(const Result &result) override { if (result.hasError()) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (_result.hasError()) { _result = TransportLatch::mergeErrorResults(_result, result); } else { @@ -109,7 +108,7 @@ public: ~SynchronizedBucketIdListResultHandler() override; void handle(const BucketIdListResult &result) override { { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); BucketIdListResultHandler::handle(result); } _latch.countDown(); @@ -161,21 +160,21 @@ BucketInfoResultHandler::~BucketInfoResultHandler() = default; PersistenceEngine::HandlerSnapshot::UP PersistenceEngine::getHandlerSnapshot() const { - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(); } PersistenceEngine::HandlerSnapshot::UP PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace) const { - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(bucketSpace); } PersistenceEngine::HandlerSnapshot::UP PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace, const DocumentId &id) const { - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(bucketSpace, id); } @@ -190,7 +189,7 @@ PersistenceEngine::PersistenceEngine(IPersistenceEngineOwner &owner, const IReso _iterators_lock(), _owner(owner), _writeFilter(writeFilter), - _clusterState(), + _clusterStates(), _extraModifiedBuckets(), _rwMutex() { @@ -207,7 +206,7 @@ IPersistenceHandler::SP PersistenceEngine::putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const IPersistenceHandler::SP &handler) { - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.putHandler(bucketSpace, docType, handler); } @@ -215,7 +214,7 @@ PersistenceEngine::putHandler(document::BucketSpace bucketSpace, const DocTypeNa IPersistenceHandler::SP PersistenceEngine::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const { - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandler(bucketSpace, docType); } @@ -224,7 +223,7 @@ IPersistenceHandler::SP PersistenceEngine::removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) { // TODO: Grab bucket list and treat them as modified - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _handlers.removeHandler(bucketSpace, docType); } @@ -273,11 +272,11 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const Result -PersistenceEngine::setClusterState(const ClusterState &calc) +PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState &calc) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - saveClusterState(calc); - HandlerSnapshot::UP snap = getHandlerSnapshot(); + saveClusterState(bucketSpace, calc); + HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); GenericResultHandler resultHandler(snap->size()); for (; snap->handlers().valid(); snap->handlers().next()) { IPersistenceHandler *handler = snap->handlers().get(); @@ -449,7 +448,7 @@ PersistenceEngine::createIterator(const Bucket &bucket, const document::FieldSet } entry->handler_sequence = HandlerSnapshot::release(std::move(*snapshot)); - LockGuard guard(_iterators_lock); + std::lock_guard<std::mutex> guard(_iterators_lock); static IteratorId id_counter(0); IteratorId id(++id_counter); _iterators[id] = entry.release(); @@ -461,28 +460,31 @@ PersistenceEngine::IterateResult PersistenceEngine::iterate(IteratorId id, uint64_t maxByteSize, Context&) const { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - LockGuard guard(_iterators_lock); - auto it = _iterators.find(id); - if (it == _iterators.end()) { - return IterateResult(Result::PERMANENT_ERROR, make_string("Unknown iterator with id %" PRIu64, id.getValue())); - } - if (it->second->in_use) { - return IterateResult(Result::TRANSIENT_ERROR, make_string("Iterator with id %" PRIu64 " is already in use", id.getValue())); + IteratorEntry *iteratorEntry; + { + std::lock_guard<std::mutex> guard(_iterators_lock); + auto it = _iterators.find(id); + if (it == _iterators.end()) { + return IterateResult(Result::PERMANENT_ERROR, make_string("Unknown iterator with id %" PRIu64, id.getValue())); + } + iteratorEntry = it->second; + if (iteratorEntry->in_use) { + return IterateResult(Result::TRANSIENT_ERROR, make_string("Iterator with id %" PRIu64 " is already in use", id.getValue())); + } + iteratorEntry->in_use = true; } - it->second->in_use = true; - guard.unlock(); - DocumentIterator &iterator = it->second->it; + DocumentIterator &iterator = iteratorEntry->it; try { IterateResult result = iterator.iterate(maxByteSize); - LockGuard guard2(_iterators_lock); - it->second->in_use = false; + std::lock_guard<std::mutex> guard(_iterators_lock); + iteratorEntry->in_use = false; return result; } catch (const std::exception & e) { IterateResult result(Result::PERMANENT_ERROR, make_string("Caught exception during visitor iterator.iterate() = '%s'", e.what())); LOG(warning, "Caught exception during visitor iterator.iterate() = '%s'", e.what()); - LockGuard guard2(_iterators_lock); - it->second->in_use = false; + std::lock_guard<std::mutex> guard(_iterators_lock); + iteratorEntry->in_use = false; return result; } } @@ -492,7 +494,7 @@ Result PersistenceEngine::destroyIterator(IteratorId id, Context&) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - LockGuard guard(_iterators_lock); + std::lock_guard<std::mutex> guard(_iterators_lock); auto it = _iterators.find(id); if (it == _iterators.end()) { return Result(); @@ -545,7 +547,7 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const typedef BucketIdListResultV MBV; MBV extraModifiedBuckets; { - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]); } HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); @@ -613,7 +615,7 @@ PersistenceEngine::destroyIterators() for (;;) { IteratorId id; { - LockGuard guard(_iterators_lock); + std::lock_guard<std::mutex> guard(_iterators_lock); if (_iterators.empty()) break; id = _iterators.begin()->first; @@ -628,26 +630,27 @@ PersistenceEngine::destroyIterators() void -PersistenceEngine::saveClusterState(const ClusterState &calc) +PersistenceEngine::saveClusterState(BucketSpace bucketSpace, const ClusterState &calc) { auto clusterState = std::make_shared<ClusterState>(calc); { - LockGuard guard(_lock); - clusterState.swap(_clusterState); + std::lock_guard<std::mutex> guard(_lock); + clusterState.swap(_clusterStates[bucketSpace]); } } PersistenceEngine::ClusterState::SP -PersistenceEngine::savedClusterState() const +PersistenceEngine::savedClusterState(BucketSpace bucketSpace) const { - LockGuard guard(_lock); - return _clusterState; + std::lock_guard<std::mutex> guard(_lock); + auto itr(_clusterStates.find(bucketSpace)); + return ((itr != _clusterStates.end()) ? itr->second : ClusterState::SP()); } void -PersistenceEngine::propagateSavedClusterState(IPersistenceHandler &handler) +PersistenceEngine::propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler) { - ClusterState::SP clusterState(savedClusterState()); + ClusterState::SP clusterState(savedClusterState(bucketSpace)); if (!clusterState) return; // Propagate saved cluster state. @@ -663,7 +666,7 @@ PersistenceEngine::grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenc BucketIdListResultHandler resultHandler; handler.handleListBuckets(resultHandler); auto result = std::make_shared<BucketIdListResult>(resultHandler.getResult()); - LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _extraModifiedBuckets[bucketSpace].push_back(result); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index b2abc7911d7..a9c15e6f2b9 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -8,9 +8,9 @@ #include <vespa/persistence/spi/abstractpersistenceprovider.h> #include <vespa/searchcore/proton/common/handlermap.hpp> #include <vespa/searchcore/proton/persistenceengine/ipersistencehandler.h> -#include <vespa/vespalib/util/sync.h> #include <mutex> #include <shared_mutex> +#include <unordered_map> namespace proton { @@ -71,12 +71,12 @@ private: const ssize_t _defaultSerializedSize; const bool _ignoreMaxBytes; PersistenceHandlerMap _handlers; - vespalib::Lock _lock; + mutable std::mutex _lock; Iterators _iterators; - vespalib::Lock _iterators_lock; + mutable std::mutex _iterators_lock; IPersistenceEngineOwner &_owner; const IResourceWriteFilter &_writeFilter; - ClusterState::SP _clusterState; + std::unordered_map<BucketSpace, ClusterState::SP, BucketSpace::hash> _clusterStates; mutable ExtraModifiedBuckets _extraModifiedBuckets; mutable std::shared_timed_mutex _rwMutex; @@ -87,8 +87,8 @@ private: HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace, const document::DocumentId &docId) const; - void saveClusterState(const ClusterState &calc); - ClusterState::SP savedClusterState() const; + void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); + ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; public: typedef std::unique_ptr<PersistenceEngine> UP; @@ -108,7 +108,7 @@ public: virtual Result initialize() override; virtual PartitionStateListResult getPartitionStates() const override; virtual BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - virtual Result setClusterState(const ClusterState& calc) override; + virtual Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; virtual Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; virtual BucketInfoResult getBucketInfo(const Bucket&) const override; virtual Result put(const Bucket&, Timestamp, const document::Document::SP&, Context&) override; @@ -129,7 +129,7 @@ public: virtual Result maintain(const Bucket&, MaintenanceLevel) override; void destroyIterators(); - void propagateSavedClusterState(IPersistenceHandler &handler); + void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler); void populateInitialBucketDB(BucketSpace bucketSpace, IPersistenceHandler &targetHandler); std::unique_lock<std::shared_timed_mutex> getWLock() const; diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp index 75a20f9f8e5..68aaad3b557 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp @@ -30,7 +30,7 @@ DocumentDBReferenceRegistry::get(vespalib::stringref name) const std::shared_ptr<IDocumentDBReference> DocumentDBReferenceRegistry::tryGet(vespalib::stringref name) const { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); auto itr = _handlers.find(name); if (itr == _handlers.end()) { return std::shared_ptr<IDocumentDBReference>(); diff --git a/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.cpp b/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.cpp index 15283c170cc..79bf970aeac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.cpp @@ -6,7 +6,6 @@ namespace proton { -using LockGuard = std::unique_lock<std::mutex>; PendingLidTracker::PendingLidTracker() : _mutex(), _cond(), @@ -19,12 +18,12 @@ PendingLidTracker::~PendingLidTracker() { void PendingLidTracker::produce(uint32_t lid) { - LockGuard guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); _pending[lid]++; } void PendingLidTracker::consume(uint32_t lid) { - LockGuard guard(_mutex); + std::lock_guard<std::mutex> guard(_mutex); auto found = _pending.find(lid); assert (found != _pending.end()); assert (found->second > 0); @@ -38,7 +37,7 @@ PendingLidTracker::consume(uint32_t lid) { void PendingLidTracker::waitForConsumedLid(uint32_t lid) { - LockGuard guard(_mutex); + std::unique_lock<std::mutex> guard(_mutex); while (_pending.find(lid) != _pending.end()) { _cond.wait(guard); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index cd016e5cfc4..6de5410470f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -568,7 +568,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, std::unique_lock<std::shared_timed_mutex> persistenceWGuard(_persistenceEngine->getWLock()); auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret); if (!_isInitializing) { - _persistenceEngine->propagateSavedClusterState(*persistenceHandler); + _persistenceEngine->propagateSavedClusterState(bucketSpace, *persistenceHandler); _persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler); } // TODO: Fix race with new cluster state setting. @@ -655,7 +655,8 @@ PrepareRestartFlushStrategy::Config createPrepareRestartConfig(const ProtonConfig &protonConfig) { return PrepareRestartFlushStrategy::Config(protonConfig.flush.preparerestart.replaycost, - protonConfig.flush.preparerestart.writecost); + protonConfig.flush.preparerestart.replayoperationcost, + protonConfig.flush.preparerestart.writecost); } } diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index 560510ce744..cda1d0103c7 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "summaryengine.h" -#include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> LOG_SETUP(".proton.summaryengine.summaryengine"); @@ -17,9 +16,7 @@ private: DocsumRequest::Source _request; public: - DocsumTask(SummaryEngine & engine, - DocsumRequest::Source request, - DocsumClient & client) + DocsumTask(SummaryEngine & engine, DocsumRequest::Source request, DocsumClient & client) : _engine(engine), _client(client), _request(std::move(request)) @@ -62,8 +59,7 @@ SummaryEngine::close() } ISearchHandler::SP -SummaryEngine::putSearchHandler(const DocTypeName &docTypeName, - const ISearchHandler::SP & searchHandler) +SummaryEngine::putSearchHandler(const DocTypeName &docTypeName, const ISearchHandler::SP & searchHandler) { vespalib::LockGuard guard(_lock); return _handlers.putHandler(docTypeName, searchHandler); @@ -72,6 +68,7 @@ SummaryEngine::putSearchHandler(const DocTypeName &docTypeName, ISearchHandler::SP SummaryEngine::getSearchHandler(const DocTypeName &docTypeName) { + vespalib::LockGuard guard(_lock); return _handlers.getHandler(docTypeName); } @@ -101,17 +98,11 @@ SummaryEngine::getDocsums(DocsumRequest::Source request, DocsumClient & client) DocsumReply::UP SummaryEngine::getDocsums(DocsumRequest::UP req) { - DocsumReply::UP reply; - reply.reset(new DocsumReply()); + DocsumReply::UP reply = std::make_unique<DocsumReply>(); if (req) { - ISearchHandler::SP searchHandler; - { // try to find the summary handler corresponding to the specified search doc type - vespalib::LockGuard guard(_lock); - DocTypeName docTypeName(*req); - searchHandler = _handlers.getHandler(docTypeName); - } - if (searchHandler.get() != NULL) { + ISearchHandler::SP searchHandler = getSearchHandler(DocTypeName(*req)); + if (searchHandler) { reply = searchHandler->getDocsums(*req); } else { vespalib::Sequence<ISearchHandler*>::UP snapshot; diff --git a/searchlib/pom.xml b/searchlib/pom.xml index bb305f460ca..8e15e0d425c 100644 --- a/searchlib/pom.xml +++ b/searchlib/pom.xml @@ -43,6 +43,16 @@ <artifactId>proto</artifactId> <version>1.4.0</version> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleCompatibleValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleCompatibleValue.java index dab89fe8955..ea750295423 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleCompatibleValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleCompatibleValue.java @@ -39,6 +39,31 @@ public abstract class DoubleCompatibleValue extends Value { } @Override + public Value modulo(Value value) { + return new DoubleValue(asDouble() % value.asDouble()); + } + + @Override + public Value and(Value value) { + return new BooleanValue(asBoolean() && value.asBoolean()); + } + + @Override + public Value or(Value value) { + return new BooleanValue(asBoolean() || value.asBoolean()); + } + + @Override + public Value not() { + return new BooleanValue(!asBoolean()); + } + + @Override + public Value power(Value value) { + return new DoubleValue(Function.pow.evaluate(asDouble(), value.asDouble())); + } + + @Override public Value compare(TruthOperator operator, Value value) { return new BooleanValue(operator.evaluate(asDouble(), value.asDouble())); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java index 28272e58c91..17157ab385f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java @@ -98,6 +98,17 @@ public final class DoubleValue extends DoubleCompatibleValue { } @Override + public Value modulo(Value value) { + try { + return mutable(this.value % value.asDouble()); + } + catch (UnsupportedOperationException e) { + throw unsupported("modulo",value); + } + } + + + @Override public Value function(Function function, Value value) { // use the tensor implementation of max and min if the argument is a tensor if ( (function.equals(Function.min) || function.equals(Function.max)) && value instanceof TensorValue) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/StringValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/StringValue.java index acf301f3b80..ac8aba6a617 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/StringValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/StringValue.java @@ -54,17 +54,42 @@ public class StringValue extends Value { @Override public Value subtract(Value value) { - throw new UnsupportedOperationException("String values ('" + value + "') does not support subtraction"); + throw new UnsupportedOperationException("String values ('" + value + "') do not support subtraction"); } @Override public Value multiply(Value value) { - throw new UnsupportedOperationException("String values ('" + value + "') does not support multiplication"); + throw new UnsupportedOperationException("String values ('" + value + "') do not support multiplication"); } @Override public Value divide(Value value) { - throw new UnsupportedOperationException("String values ('" + value + "') does not support division"); + throw new UnsupportedOperationException("String values ('" + value + "') do not support division"); + } + + @Override + public Value modulo(Value value) { + throw new UnsupportedOperationException("String values ('" + value + "') do not support modulo"); + } + + @Override + public Value and(Value value) { + throw new UnsupportedOperationException("String values ('" + value + "') do not support and"); + } + + @Override + public Value or(Value value) { + throw new UnsupportedOperationException("String values ('" + value + "') do not support or"); + } + + @Override + public Value not() { + throw new UnsupportedOperationException("String values ('" + value + "') do not support not"); + } + + @Override + public Value power(Value value) { + throw new UnsupportedOperationException("String values ('" + value + "') do not support ^"); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java index 6cf15837da1..49c3ccb7b01 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java @@ -81,6 +81,43 @@ public class TensorValue extends Value { return new TensorValue(value.map((value) -> value / argument.asDouble())); } + @Override + public Value modulo(Value argument) { + if (argument instanceof TensorValue) + return new TensorValue(value.fmod(((TensorValue) argument).value)); + else + return new TensorValue(value.map((value) -> value % argument.asDouble())); + } + + @Override + public Value and(Value argument) { + if (argument instanceof TensorValue) + return new TensorValue(value.join(((TensorValue)argument).value, (a, b) -> ((a!=0.0) && (b!=0.0)) ? 1.0 : 0.0 )); + else + return new TensorValue(value.map((value) -> ((value!=0.0) && argument.asBoolean()) ? 1 : 0)); + } + + @Override + public Value or(Value argument) { + if (argument instanceof TensorValue) + return new TensorValue(value.join(((TensorValue)argument).value, (a, b) -> ((a!=0.0) || (b!=0.0)) ? 1.0 : 0.0 )); + else + return new TensorValue(value.map((value) -> ((value!=0.0) || argument.asBoolean()) ? 1 : 0)); + } + + @Override + public Value not() { + return new TensorValue(value.map((value) -> (value==0.0) ? 1.0 : 0.0)); + } + + @Override + public Value power(Value argument) { + if (argument instanceof TensorValue) + return new TensorValue(value.pow(((TensorValue)argument).value)); + else + return new TensorValue(value.map((value) -> Math.pow(value, argument.asDouble()))); + } + private Tensor asTensor(Value value, String operationName) { if ( ! (value instanceof TensorValue)) throw new UnsupportedOperationException("Could not perform " + operationName + @@ -103,6 +140,7 @@ public class TensorValue extends Value { case SMALLEREQUAL: return value.smallerOrEqual(argument); case EQUAL: return value.equal(argument); case NOTEQUAL: return value.notEqual(argument); + case APPROX_EQUAL: return value.approxEqual(argument); default: throw new UnsupportedOperationException("Tensors cannot be compared with " + operator); } } @@ -120,6 +158,9 @@ public class TensorValue extends Value { case min: return value.min(argument); case max: return value.max(argument); case atan2: return value.atan2(argument); + case pow: return value.pow(argument); + case fmod: return value.fmod(argument); + case ldexp: return value.ldexp(argument); default: throw new UnsupportedOperationException("Cannot combine two tensors using " + function); } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index a63387506a0..b2ccbe572d0 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -41,6 +41,16 @@ public abstract class Value { public abstract Value divide(Value value); + public abstract Value modulo(Value value); + + public abstract Value and(Value value); + + public abstract Value or(Value value); + + public abstract Value not(); + + public abstract Value power(Value value); + /** Perform the comparison specified by the operator between this value and the given value */ public abstract Value compare(TruthOperator operator, Value value); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java index 91d8abec1be..518a15bcc87 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java @@ -77,7 +77,7 @@ public final class ArithmeticNode extends CompositeNode { Iterator<ExpressionNode> child = children.iterator(); Deque<ValueItem> stack = new ArrayDeque<>(); - stack.push(new ValueItem(ArithmeticOperator.PLUS, child.next().evaluate(context))); + stack.push(new ValueItem(ArithmeticOperator.OR, child.next().evaluate(context))); for (Iterator<ArithmeticOperator> it = operators.iterator(); it.hasNext() && child.hasNext();) { ArithmeticOperator op = it.next(); if (!stack.isEmpty()) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticOperator.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticOperator.java index 5a5237c2608..a715490e95a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticOperator.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticOperator.java @@ -14,17 +14,29 @@ import java.util.List; */ public enum ArithmeticOperator { - PLUS(0, "+") { public Value evaluate(Value x, Value y) { + OR(0, "||") { public Value evaluate(Value x, Value y) { + return x.or(y); + }}, + AND(1, "&&") { public Value evaluate(Value x, Value y) { + return x.and(y); + }}, + PLUS(2, "+") { public Value evaluate(Value x, Value y) { return x.add(y); }}, - MINUS(1, "-") { public Value evaluate(Value x, Value y) { + MINUS(3, "-") { public Value evaluate(Value x, Value y) { return x.subtract(y); }}, - MULTIPLY(2, "*") { public Value evaluate(Value x, Value y) { + MULTIPLY(4, "*") { public Value evaluate(Value x, Value y) { return x.multiply(y); }}, - DIVIDE(3, "/") { public Value evaluate(Value x, Value y) { + DIVIDE(5, "/") { public Value evaluate(Value x, Value y) { return x.divide(y); + }}, + MODULO(6, "%") { public Value evaluate(Value x, Value y) { + return x.modulo(y); + }}, + POWER(7, "^") { public Value evaluate(Value x, Value y) { + return x.power(y); }}; /** A list of all the operators in this in order of decreasing precedence */ @@ -52,10 +64,14 @@ public enum ArithmeticOperator { private static List<ArithmeticOperator> operatorsByPrecedence() { List<ArithmeticOperator> operators = new ArrayList<>(); + operators.add(POWER); + operators.add(MODULO); operators.add(DIVIDE); operators.add(MULTIPLY); operators.add(MINUS); operators.add(PLUS); + operators.add(AND); + operators.add(OR); return Collections.unmodifiableList(operators); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java index fc4a511b307..c3c1c371a68 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java @@ -39,7 +39,7 @@ public enum Function implements Serializable { atan2(2) { public double evaluate(double x, double y) { return atan2(x,y); } }, fmod(2) { public double evaluate(double x, double y) { return x % y; } }, - ldexp(2) { public double evaluate(double x, double y) { return x*pow(2,y); } }, + ldexp(2) { public double evaluate(double x, double y) { return x*pow(2,(int)y); } }, max(2) { public double evaluate(double x, double y) { return max(x,y); } }, min(2) { public double evaluate(double x, double y) { return min(x,y); } }, pow(2) { public double evaluate(double x, double y) { return pow(x,y); } }; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java new file mode 100644 index 00000000000..8c459a032bd --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchlib.rankingexpression.rule; + +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; + +import java.util.Collections; +import java.util.Deque; +import java.util.List; + +/** + * A node which flips the logical value produced from the nested expression. + * + * @author lesters + */ +public class NotNode extends BooleanNode { + + private final ExpressionNode value; + + public NotNode(ExpressionNode value) { + this.value = value; + } + + public ExpressionNode getValue() { + return value; + } + + @Override + public List<ExpressionNode> children() { + return Collections.singletonList(value); + } + + @Override + public String toString(SerializationContext context, Deque<String> path, CompositeNode parent) { + return "!" + value.toString(context, path, parent); + } + + @Override + public Value evaluate(Context context) { + return value.evaluate(context).not(); + } + + @Override + public NotNode setChildren(List<ExpressionNode> children) { + if (children.size() != 1) throw new IllegalArgumentException("Expected 1 children but got " + children.size()); + return new NotNode(children.get(0)); + } + +} + diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java index f8e44f1087c..f6b1a1a8979 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java @@ -4,9 +4,14 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.tensor.Tensor; -import java.util.*; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.function.Predicate; /** * A node which returns true or false depending on a set membership test @@ -55,11 +60,30 @@ public class SetMembershipNode extends BooleanNode { @Override public Value evaluate(Context context) { Value value = testValue.evaluate(context); + if (value instanceof TensorValue) { + return evaluateTensor(((TensorValue) value).asTensor(), context); + } + return evaluateValue(value, context); + } + + private Value evaluateValue(Value value, Context context) { + return new BooleanValue(testMembership(value::equals, context)); + } + + private Value evaluateTensor(Tensor tensor, Context context) { + return new TensorValue(tensor.map((value) -> contains(value, context) ? 1.0 : 0.0)); + } + + private boolean contains(double value, Context context) { + return testMembership((setValue) -> setValue.asDouble() == value, context); + } + + private boolean testMembership(Predicate<Value> test, Context context) { for (ExpressionNode setValue : setValues) { - if (setValue.evaluate(context).equals(value)) - return new BooleanValue(true); + if (test.test(setValue.evaluate(context))) + return true; } - return new BooleanValue(false); + return false; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java index ede7c861d98..ebad0d5c21f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/Simplifier.java @@ -94,7 +94,7 @@ public class Simplifier extends ExpressionTransformer { private ExpressionNode transformIf(IfNode node) { if ( ! isConstant(node.getCondition())) return node; - if (((BooleanValue)node.getCondition().evaluate(null)).asBoolean()) + if ((node.getCondition().evaluate(null)).asBoolean()) return node.getTrueExpression(); else return node.getFalseExpression(); diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index c3b9235cc93..7821ab88b86 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -65,6 +65,8 @@ TOKEN : <DIV: "/"> | <MUL: "*"> | <DOT: "."> | + <MOD: "%"> | + <POWOP: "^"> | <DOLLAR: "$"> | <COMMA: ","> | @@ -85,6 +87,10 @@ TOKEN : <IN: "in"> | <F: "f"> | + <NOT: "!"> | + <AND: "&&"> | + <OR: "||"> | + <ABS: "abs"> | <ACOS: "acos"> | <ASIN: "asin"> | @@ -199,10 +205,14 @@ ExpressionNode arithmeticExpression() : ArithmeticOperator arithmetic() : { } { - ( <ADD> { return ArithmeticOperator.PLUS; } | - <SUB> { return ArithmeticOperator.MINUS; } | - <DIV> { return ArithmeticOperator.DIVIDE; } | - <MUL> { return ArithmeticOperator.MULTIPLY; } ) + ( <ADD> { return ArithmeticOperator.PLUS; } | + <SUB> { return ArithmeticOperator.MINUS; } | + <DIV> { return ArithmeticOperator.DIVIDE; } | + <MUL> { return ArithmeticOperator.MULTIPLY; } | + <MOD> { return ArithmeticOperator.MODULO; } | + <AND> { return ArithmeticOperator.AND; } | + <OR> { return ArithmeticOperator.OR; } | + <POWOP> { return ArithmeticOperator.POWER; } ) { return null; } } @@ -222,16 +232,23 @@ ExpressionNode value() : { ExpressionNode ret; boolean neg = false; + boolean not = false; } { - ( [ LOOKAHEAD(2) <SUB> { neg = true; } ] - ( ret = constantPrimitive() | - LOOKAHEAD(2) ret = ifExpression() | - LOOKAHEAD(4) ret = function() | - ret = feature() | - ret = queryFeature() | + ( + [ <NOT> { not = true; } ] + [ LOOKAHEAD(2) <SUB> { neg = true; } ] + ( ret = constantPrimitive() | + LOOKAHEAD(2) ret = ifExpression() | + LOOKAHEAD(4) ret = function() | + ret = feature() | + ret = queryFeature() | ( <LBRACE> ret = expression() <RBRACE> { ret = new EmbracedNode(ret); } ) ) ) - { return neg ? new NegativeNode(ret) : ret; } + { + ret = not ? new NotNode(ret) : ret; + ret = neg ? new NegativeNode(ret) : ret; + return ret; + } } IfNode ifExpression() : diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 5d357777657..82e5d0cfe5b 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -29,6 +29,7 @@ public class EvaluationTestCase { tester.assertEvaluates(0.75, "0.5 + 0.25"); tester.assertEvaluates(0.75, "one_half + a_quarter"); tester.assertEvaluates(1.25, "0.5 - 0.25 + one"); + tester.assertEvaluates(9.0, "3 ^ 2"); // String tester.assertEvaluates(1, "if(\"a\"==\"a\",1,0)"); @@ -37,6 +38,9 @@ public class EvaluationTestCase { tester.assertEvaluates(26, "2*3+4*5"); tester.assertEvaluates(1, "2/6+4/6"); tester.assertEvaluates(2 * 3 * 4 + 3 * 4 * 5 - 4 * 200 / 10, "2*3*4+3*4*5-4*200/10"); + tester.assertEvaluates(3, "1 + 10 % 6 / 2"); + tester.assertEvaluates(10.0, "3 ^ 2 + 1"); + tester.assertEvaluates(18.0, "2 * 3 ^ 2"); // Conditionals tester.assertEvaluates(2 * (3 * 4 + 3) * (4 * 5 - 4 * 200) / 10, "2*(3*4+3)*(4*5-4*200)/10"); @@ -89,6 +93,38 @@ public class EvaluationTestCase { } @Test + public void testBooleanEvaluation() { + EvaluationTester tester = new EvaluationTester(); + + // and + tester.assertEvaluates(false, "0 && 0"); + tester.assertEvaluates(false, "0 && 1"); + tester.assertEvaluates(false, "1 && 0"); + tester.assertEvaluates(true, "1 && 1"); + tester.assertEvaluates(true, "1 && 2"); + tester.assertEvaluates(true, "1 && 0.1"); + + // or + tester.assertEvaluates(false, "0 || 0"); + tester.assertEvaluates(true, "0 || 0.1"); + tester.assertEvaluates(true, "0 || 1"); + tester.assertEvaluates(true, "1 || 0"); + tester.assertEvaluates(true, "1 || 1"); + + // not + tester.assertEvaluates(true, "!0"); + tester.assertEvaluates(false, "!1"); + tester.assertEvaluates(false, "!2"); + tester.assertEvaluates(true, "!0 && 1"); + + // precedence + tester.assertEvaluates(0, "2 * (0 && 1)"); + tester.assertEvaluates(2, "2 * (1 && 1)"); + tester.assertEvaluates(true, "2 + 0 && 1"); + tester.assertEvaluates(true, "1 && 0 + 2"); + } + + @Test public void testTensorEvaluation() { EvaluationTester tester = new EvaluationTester(); tester.assertEvaluates("{}", "tensor0", "{}"); @@ -107,6 +143,16 @@ public class EvaluationTestCase { "min(tensor0, 0)", "{ {d1:0}:-10, {d1:1}:0, {d1:2}:10 }"); tester.assertEvaluates("{ {d1:0}:0, {d1:1}:0, {d1:2 }:10 }", "max(tensor0, 0)", "{ {d1:0}:-10, {d1:1}:0, {d1:2}:10 }"); + // operators + tester.assertEvaluates("{ {d1:0}:1, {d1:1}:1, {d1:2 }:1 }", + "tensor0 % 2 == map(tensor0, f(x) (x % 2))", "{ {d1:0}:2, {d1:1}:3, {d1:2}:4 }"); + tester.assertEvaluates("{ {d1:0}:1, {d1:1}:1, {d1:2 }:1 }", + "tensor0 || 1 == map(tensor0, f(x) (x || 1))", "{ {d1:0}:2, {d1:1}:3, {d1:2}:4 }"); + tester.assertEvaluates("{ {d1:0}:1, {d1:1}:1, {d1:2 }:1 }", + "tensor0 && 1 == map(tensor0, f(x) (x && 1))", "{ {d1:0}:2, {d1:1}:3, {d1:2}:4 }"); + tester.assertEvaluates("{ {d1:0}:1, {d1:1}:1, {d1:2 }:1 }", + "!tensor0 == map(tensor0, f(x) (!x))", "{ {d1:0}:0, {d1:1}:1, {d1:2}:0 }"); + // -- explicitly implemented functions (not foolproof tests as we don't bother testing float value equivalence) tester.assertEvaluates("{ {x:0}:1, {x:1}:2 }", "abs(tensor0)", "{ {x:0}:1, {x:1}:-2 }"); tester.assertEvaluates("{ {x:0}:0, {x:1}:0 }", "acos(tensor0)", "{ {x:0}:1, {x:1}:1 }"); @@ -122,8 +168,9 @@ public class EvaluationTestCase { tester.assertEvaluates("{ {x:0}:0, {x:1}:0 }", "isNan(tensor0)", "{ {x:0}:1, {x:1}:2 }"); tester.assertEvaluates("{ {x:0}:0, {x:1}:0 }", "log(tensor0)", "{ {x:0}:1, {x:1}:1 }"); tester.assertEvaluates("{ {x:0}:0, {x:1}:1 }", "log10(tensor0)", "{ {x:0}:1, {x:1}:10 }"); - tester.assertEvaluates("{ {x:0}:0, {x:1}:2 }", "fmod(tensor0, 3)", "{ {x:0}:3, {x:1}:8 }"); + tester.assertEvaluates("{ {x:0}:0, {x:1}:2 }", "fmod(tensor0, 3)","{ {x:0}:3, {x:1}:8 }"); tester.assertEvaluates("{ {x:0}:1, {x:1}:8 }", "pow(tensor0, 3)", "{ {x:0}:1, {x:1}:2 }"); + tester.assertEvaluates("{ {x:0}:8, {x:1}:16 }", "ldexp(tensor0,3.1)","{ {x:0}:1, {x:1}:2 }"); tester.assertEvaluates("{ {x:0}:1, {x:1}:2 }", "relu(tensor0)", "{ {x:0}:1, {x:1}:2 }"); tester.assertEvaluates("{ {x:0}:1, {x:1}:2 }", "round(tensor0)", "{ {x:0}:1, {x:1}:1.8 }"); tester.assertEvaluates("{ {x:0}:0.5, {x:1}:0.5 }", "sigmoid(tensor0)","{ {x:0}:0, {x:1}:0 }"); @@ -201,6 +248,16 @@ public class EvaluationTestCase { "max(tensor0, tensor1)", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); tester.assertEvaluates("{ {x:0,y:0}:3, {x:1,y:0}:5 }", "min(tensor0, tensor1)", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:243, {x:1,y:0}:16807 }", + "pow(tensor0, tensor1)", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:243, {x:1,y:0}:16807 }", + "tensor0 ^ tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:3, {x:1,y:0}:2 }", + "fmod(tensor0, tensor1)", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:3, {x:1,y:0}:2 }", + "tensor0 % tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:96, {x:1,y:0}:224 }", + "ldexp(tensor0, tensor1)", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5.1 }"); tester.assertEvaluates("{ {x:0,y:0,z:0}:7, {x:0,y:0,z:1}:13, {x:1,y:0,z:0}:21, {x:1,y:0,z:1}:39, {x:0,y:1,z:0}:55, {x:0,y:1,z:1}:0, {x:1,y:1,z:0}:0, {x:1,y:1,z:1}:0 }", "tensor0 * tensor1", "{ {x:0,y:0}:1, {x:1,y:0}:3, {x:0,y:1}:5, {x:1,y:1}:0 }", "{ {y:0,z:0}:7, {y:1,z:0}:11, {y:0,z:1}:13, {y:1,z:1}:0 }"); tester.assertEvaluates("{ {x:0,y:1,z:0}:35, {x:0,y:1,z:1}:65 }", @@ -225,8 +282,13 @@ public class EvaluationTestCase { "tensor0 <= tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); tester.assertEvaluates("{ {x:0,y:0}:0, {x:1,y:0}:1 }", "tensor0 == tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:7 }"); + tester.assertEvaluates("{ {x:0,y:0}:0, {x:1,y:0}:1 }", + "tensor0 ~= tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:7 }"); tester.assertEvaluates("{ {x:0,y:0}:1, {x:1,y:0}:0 }", "tensor0 != tensor1", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:7 }"); + tester.assertEvaluates("{ {x:0}:1, {x:1}:0 }", + "tensor0 in [1,2,3]", "{ {x:0}:3, {x:1}:7 }"); + // TODO // argmax // argmin diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTester.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTester.java index d67c9dfd9dc..ee2b1c147e3 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTester.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTester.java @@ -58,10 +58,18 @@ public class EvaluationTester { return assertEvaluates(value, expressionString, defaultContext); } + public RankingExpression assertEvaluates(boolean value, String expressionString) { + return assertEvaluates(value, expressionString, defaultContext); + } + public RankingExpression assertEvaluates(double value, String expressionString, Context context) { return assertEvaluates(new DoubleValue(value), expressionString, context, ""); } + public RankingExpression assertEvaluates(boolean value, String expressionString, Context context) { + return assertEvaluates(new BooleanValue(value), expressionString, context, ""); + } + public RankingExpression assertEvaluates(Value value, String expressionString, Context context, String explanation) { try { RankingExpression expression = new RankingExpression(expressionString); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/tensor/TensorConformanceTest.java b/searchlib/src/test/java/com/yahoo/searchlib/tensor/TensorConformanceTest.java new file mode 100644 index 00000000000..dde9d4bf21e --- /dev/null +++ b/searchlib/src/test/java/com/yahoo/searchlib/tensor/TensorConformanceTest.java @@ -0,0 +1,138 @@ +package com.yahoo.searchlib.tensor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleCompatibleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +public class TensorConformanceTest { + + private static String testPath = "eval/src/apps/tensor_conformance/test_spec.json"; + + @Test + public void testConformance() throws IOException { + File testSpec = new File(testPath); + if (!testSpec.exists()) { + testSpec = new File("../" + testPath); + } + int count = 0; + List<Integer> failList = new ArrayList<>(); + + try(BufferedReader br = new BufferedReader(new FileReader(testSpec))) { + String test = br.readLine(); + while (test != null) { + boolean success = testCase(test, count); + if (!success) { + failList.add(count); + } + test = br.readLine(); + count++; + } + } + assertEquals(failList.size() + " conformance test fails: " + failList, 0, failList.size()); + } + + private boolean testCase(String test, int count) { + try { + ObjectMapper mapper = new ObjectMapper(); + JsonNode node = mapper.readTree(test); + + if (node.has("num_tests")) { + Assert.assertEquals(node.get("num_tests").asInt(), count); + return true; + } + if (!node.has("expression")) { + return true; // ignore + } + + String expression = node.get("expression").asText(); + MapContext context = getInput(node.get("inputs")); + Tensor expect = getTensor(node.get("result").get("expect").asText()); + Tensor result = evaluate(expression, context); + boolean equals = Tensor.equals(result, expect); + if (!equals) { + System.out.println(count + " : Tensors not equal. Result: " + result.toString() + " Expected: " + expect.toString() + " -> expression \"" + expression + "\""); + } + return equals; + + } catch (Exception e) { + System.out.println(count + " : " + e.toString()); + } + return false; + } + + private Tensor evaluate(String expression, MapContext context) throws ParseException { + Value value = new RankingExpression(expression).evaluate(context); + if (!(value instanceof TensorValue)) { + throw new IllegalArgumentException("Result is not a tensor"); + } + return ((TensorValue)value).asTensor(); + } + + private MapContext getInput(JsonNode inputs) { + MapContext context = new MapContext(); + for (Iterator<String> i = inputs.fieldNames(); i.hasNext(); ) { + String name = i.next(); + String value = inputs.get(name).asText(); + Tensor tensor = getTensor(value); + context.put(name, new TensorValue(tensor)); + } + return context; + } + + private Tensor getTensor(String binaryRepresentation) { + byte[] bin = getBytes(binaryRepresentation); + return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(bin)); + } + + private byte[] getBytes(String binaryRepresentation) { + return parseHexValue(binaryRepresentation.substring(2)); + } + + private byte[] parseHexValue(String s) { + final int len = s.length(); + byte[] bytes = new byte[len/2]; + for (int i = 0; i < len; i += 2) { + int c1 = hexValue(s.charAt(i)) << 4; + int c2 = hexValue(s.charAt(i + 1)); + bytes[i/2] = (byte)(c1 + c2); + } + return bytes; + } + + private int hexValue(Character c) { + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } else if (c >= '0' && c <= '9') { + return c - '0'; + } + throw new IllegalArgumentException("Hex contains illegal characters"); + } + +} + diff --git a/searchlib/src/tests/features/constant/constant_test.cpp b/searchlib/src/tests/features/constant/constant_test.cpp index a10f76e25ba..4a88fde58ce 100644 --- a/searchlib/src/tests/features/constant/constant_test.cpp +++ b/searchlib/src/tests/features/constant/constant_test.cpp @@ -19,7 +19,6 @@ using namespace search::features; using vespalib::eval::Function; using vespalib::eval::Value; using vespalib::eval::DoubleValue; -using vespalib::eval::TensorValue; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; @@ -39,7 +38,7 @@ Tensor::UP createTensor(const TensorCells &cells, } Tensor::UP make_tensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().create(spec); + auto tensor = DefaultTensorEngine::ref().from_spec(spec); return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } @@ -80,7 +79,7 @@ struct ExecFixture ValueType type(tensor->getType()); test.getIndexEnv().addConstantValue(name, std::move(type), - std::make_unique<TensorValue>(std::move(tensor))); + std::move(tensor)); } void addDouble(const vespalib::string &name, const double value) { diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index be7bb9defac..b097f27342d 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -54,7 +54,7 @@ Tensor::UP createTensor(const TensorCells &cells, } Tensor::UP make_tensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().create(spec); + auto tensor = DefaultTensorEngine::ref().from_spec(spec); return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } diff --git a/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp b/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp index 0a900ad9ec8..1ac524b5d0b 100644 --- a/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp +++ b/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp @@ -36,7 +36,7 @@ typedef search::AttributeVector::SP AttributePtr; typedef FtTestApp FTA; Tensor::UP make_tensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().create(spec); + auto tensor = DefaultTensorEngine::ref().from_spec(spec); return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } diff --git a/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp b/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp index cad0c56b0ca..e0eee954a53 100644 --- a/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp +++ b/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp @@ -37,7 +37,7 @@ typedef search::AttributeVector::SP AttributePtr; typedef FtTestApp FTA; Tensor::UP make_tensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().create(spec); + auto tensor = DefaultTensorEngine::ref().from_spec(spec); return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } diff --git a/searchlib/src/tests/postinglistbm/andstress.cpp b/searchlib/src/tests/postinglistbm/andstress.cpp index 736d53508b4..40f919509e8 100644 --- a/searchlib/src/tests/postinglistbm/andstress.cpp +++ b/searchlib/src/tests/postinglistbm/andstress.cpp @@ -280,7 +280,7 @@ AndStressMaster::Task * AndStressMaster::getTask() { Task *result = NULL; - std::unique_lock<std::mutex> taskGuard(_taskLock); + std::lock_guard<std::mutex> taskGuard(_taskLock); if (_taskIdx < _tasks.size()) { result = &_tasks[_taskIdx]; ++_taskIdx; diff --git a/searchlib/src/tests/rankingexpression/rankingexpressionlist b/searchlib/src/tests/rankingexpression/rankingexpressionlist index 327f2b161cd..77b2294c668 100644 --- a/searchlib/src/tests/rankingexpression/rankingexpressionlist +++ b/searchlib/src/tests/rankingexpression/rankingexpressionlist @@ -160,3 +160,7 @@ mysum ( mysum(4, 4), value( 4 ), value(4) ); mysum(mysum(4,4),value(4),value(4) "1008\x1977" "100819\x77" if(1.09999~=1.1,2,3); if (1.09999 ~= 1.1, 2, 3) +10 % 3 +1 && 0 || 1 +!a && (a || a) +10 ^ 3 diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 28b4ad3c4e4..2e88f0e90b0 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -21,7 +21,7 @@ using EntryRef = DenseTensorStore::EntryRef; Tensor::UP makeTensor(const TensorSpec &spec) { - auto tensor = DefaultTensorEngine::ref().create(spec); + auto tensor = DefaultTensorEngine::ref().from_spec(spec); return Tensor::UP(dynamic_cast<Tensor *>(tensor.release())); } diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 1a0e425e0ef..43ce48282ee 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -18,10 +18,10 @@ namespace features { class ConstantTensorExecutor : public fef::FeatureExecutor { private: - const vespalib::eval::TensorValue::UP _tensor; + vespalib::eval::Value::UP _tensor; public: - ConstantTensorExecutor(vespalib::eval::TensorValue::UP tensor) + ConstantTensorExecutor(vespalib::eval::Value::UP tensor) : _tensor(std::move(tensor)) {} virtual bool isPure() override { return true; } @@ -29,11 +29,12 @@ public: outputs().set_object(0, *_tensor); } static fef::FeatureExecutor &create(std::unique_ptr<vespalib::eval::Tensor> tensor, vespalib::Stash &stash) { - return stash.create<ConstantTensorExecutor>(std::make_unique<vespalib::eval::TensorValue>(std::move(tensor))); + return stash.create<ConstantTensorExecutor>(std::move(tensor)); } static fef::FeatureExecutor &createEmpty(const vespalib::eval::ValueType &valueType, vespalib::Stash &stash) { - return create(vespalib::tensor::DefaultTensorEngine::ref() - .create(vespalib::eval::TensorSpec(valueType.to_spec())), stash); + const auto &engine = vespalib::tensor::DefaultTensorEngine::ref(); + auto spec = vespalib::eval::TensorSpec(valueType.to_spec()); + return stash.create<ConstantTensorExecutor>(engine.from_spec(spec)); } static fef::FeatureExecutor &createEmpty(vespalib::Stash &stash) { return createEmpty(vespalib::eval::ValueType::double_type(), stash); diff --git a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp index 76252486bf4..487bc724e07 100644 --- a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp +++ b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp @@ -5,7 +5,6 @@ using search::tensor::DenseTensorAttribute; using vespalib::eval::Tensor; -using vespalib::eval::TensorValue; using vespalib::tensor::MutableDenseTensorView; namespace search { @@ -14,8 +13,7 @@ namespace features { DenseTensorAttributeExecutor:: DenseTensorAttributeExecutor(const DenseTensorAttribute *attribute) : _attribute(attribute), - _tensorView(_attribute->getConfig().tensorType()), - _tensor(_tensorView) + _tensorView(_attribute->getConfig().tensorType()) { } @@ -23,7 +21,7 @@ void DenseTensorAttributeExecutor::execute(uint32_t docId) { _attribute->getTensor(docId, _tensorView); - outputs().set_object(0, _tensor); + outputs().set_object(0, _tensorView); } } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h index 68042075942..ac3d327c12a 100644 --- a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.h @@ -19,7 +19,6 @@ class DenseTensorAttributeExecutor : public fef::FeatureExecutor private: const search::tensor::DenseTensorAttribute *_attribute; vespalib::tensor::MutableDenseTensorView _tensorView; - vespalib::eval::TensorValue _tensor; public: DenseTensorAttributeExecutor(const search::tensor::DenseTensorAttribute *attribute); diff --git a/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp b/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp index 6ee7664f2bb..03393d6f590 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp @@ -3,8 +3,6 @@ #include "tensor_attribute_executor.h" #include <vespa/searchlib/tensor/tensor_attribute.h> -using vespalib::eval::TensorValue; - namespace search { namespace features { @@ -12,20 +10,19 @@ TensorAttributeExecutor:: TensorAttributeExecutor(const search::tensor::TensorAttribute *attribute) : _attribute(attribute), _emptyTensor(attribute->getEmptyTensor()), - _tensor(*_emptyTensor) + _tensor() { } void TensorAttributeExecutor::execute(uint32_t docId) { - auto tensor = _attribute->getTensor(docId); - if (!tensor) { - _tensor = TensorValue(*_emptyTensor); + _tensor = _attribute->getTensor(docId); + if (_tensor) { + outputs().set_object(0, *_tensor); } else { - _tensor = TensorValue(std::move(tensor)); + outputs().set_object(0, *_emptyTensor); } - outputs().set_object(0, _tensor); } } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.h b/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.h index 198b03e3d1d..0f1e21c8cad 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.h @@ -17,7 +17,7 @@ class TensorAttributeExecutor : public fef::FeatureExecutor private: const search::tensor::TensorAttribute *_attribute; std::unique_ptr<vespalib::eval::Tensor> _emptyTensor; - vespalib::eval::TensorValue _tensor; + std::unique_ptr<vespalib::eval::Tensor> _tensor; public: TensorAttributeExecutor(const search::tensor::TensorAttribute *attribute); diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h b/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h index 31b92f89538..f102749f1b6 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h @@ -22,7 +22,7 @@ private: const search::attribute::IAttributeVector *_attribute; vespalib::string _dimension; WeightedBufferType _attrBuffer; - vespalib::eval::TensorValue::UP _tensor; + std::unique_ptr<vespalib::tensor::Tensor> _tensor; public: TensorFromAttributeExecutor(const search::attribute::IAttributeVector *attribute, @@ -48,7 +48,7 @@ TensorFromAttributeExecutor<WeightedBufferType>::execute(uint32_t docId) builder.add_label(dimensionEnum, vespalib::string(_attrBuffer[i].value())); builder.add_cell(_attrBuffer[i].weight()); } - _tensor = vespalib::eval::TensorValue::UP(new vespalib::eval::TensorValue(builder.build())); + _tensor = builder.build(); outputs().set_object(0, *_tensor); } diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp index 19d744dfd28..b63a54785a4 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp @@ -27,32 +27,16 @@ MultiSearch::remove(size_t index) void MultiSearch::doUnpack(uint32_t docid) { - size_t sz(_children.size()); - for (size_t i = 0; i < sz; ) { - if (__builtin_expect(_children[i]->getDocId() < docid, false)) { - _children[i]->doSeek(docid); - if (_children[i]->isAtEnd()) { - sz = deactivate(i); - continue; - } + for (SearchIterator *child: _children) { + if (__builtin_expect(child->getDocId() < docid, false)) { + child->doSeek(docid); } - if (__builtin_expect(_children[i]->getDocId() == docid, false)) { - _children[i]->doUnpack(docid); + if (__builtin_expect(child->getDocId() == docid, false)) { + child->doUnpack(docid); } - i++; } } -size_t -MultiSearch::deactivate(size_t idx) -{ - assert(idx < _children.size()); - delete _children[idx]; - _children[idx] = _children.back(); - _children.pop_back(); - return _children.size(); -} - MultiSearch::MultiSearch(const Children & children) : _children(children) { diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.h b/searchlib/src/vespa/searchlib/queryeval/multisearch.h index 16bbd5d4ecc..d67f895ddb5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.h @@ -54,7 +54,6 @@ private: virtual void onInsert(size_t index) { (void) index; } bool isMultiSearch() const override { return true; } - size_t deactivate(size_t index); Children _children; }; diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.cpp b/staging_vespalib/src/vespa/vespalib/util/clock.cpp index b19b067afa9..c9768417914 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/clock.cpp @@ -45,7 +45,7 @@ void Clock::Run(FastOS_ThreadInterface *thread, void *arguments) void Clock::stop(void) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _stop = true; _cond.notify_all(); } diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java index d03a08a27db..baf0b41b270 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java @@ -19,24 +19,15 @@ import org.osgi.framework.Bundle; import org.osgi.framework.BundleActivator; import org.osgi.framework.BundleContext; -import java.io.FileInputStream; import java.io.IOException; import java.net.InetSocketAddress; -import java.nio.channels.FileChannel; import java.nio.channels.ServerSocketChannel; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Hashtable; import java.util.List; -import java.util.Map; -import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toMap; /** * @author Einar M R Rosenvinge @@ -45,69 +36,19 @@ public class StandaloneContainerActivator implements BundleActivator { @Override public void start(BundleContext bundleContext) throws Exception { - Container container = getContainer(); - List<ConnectorConfig> connectorConfigs = getConnectorConfigs(container); - - Stream<Path> keyStorePaths = getKeyStorePaths(connectorConfigs); - Map<Path, FileChannel> fileChannels = openFiles(keyStorePaths); - registerKeyStoreFileChannels(bundleContext, fileChannels); - - for (ConnectorConfig config: connectorConfigs) { + for (ConnectorConfig config: getConnectorConfigs(getContainer())) { ServerSocketChannel socketChannel = bindChannel(config); registerChannels(bundleContext, config.listenPort(), socketChannel); } } - private void registerKeyStoreFileChannels(BundleContext bundleContext, Map<Path, FileChannel> fileChannels) { - Hashtable<String, Object> properties = new Hashtable<>(); - properties.put("role", "com.yahoo.container.standalone.StandaloneContainerActivator.KeyStoreFileChannels"); - //Since Standalone container and jdisc http service don't have a suitable common module for placing a wrapper class for fileChannels, - //we register it with the type map. In the future, we should wrap this. - bundleContext.registerService(Map.class, - Collections.unmodifiableMap(fileChannels), - properties); - } - - private Map<Path, FileChannel> openFiles(Stream<Path> keyStorePaths) { - return keyStorePaths.collect(toMap( - Function.<Path>identity(), - StandaloneContainerActivator::getFileChannel)); - } - - private static FileChannel getFileChannel(Path path) { - try { - FileInputStream inputStream = new FileInputStream(path.toFile()); - //don't close the inputStream, as that will close the underlying channel. - return inputStream.getChannel(); - } catch (IOException e) { - throw new RuntimeException("Failed opening path " + path, e); - } - } - - private Stream<Path> getKeyStorePaths(List<ConnectorConfig> connectorConfigs) { - return connectorConfigs.stream(). - map(ConnectorConfig::ssl). - flatMap(StandaloneContainerActivator::getKeyStorePaths); - } - - private static Stream<Path> getKeyStorePaths(ConnectorConfig.Ssl ssl) { - Stream<String> paths = Stream.of( - ssl.keyStorePath(), - ssl.pemKeyStore().certificatePath(), - ssl.pemKeyStore().keyPath()); - - return paths. - filter(path -> !path.isEmpty()). - map(Paths::get); - } - - void registerChannels(BundleContext bundleContext, int listenPort, ServerSocketChannel boundChannel) { + static void registerChannels(BundleContext bundleContext, int listenPort, ServerSocketChannel boundChannel) { Hashtable<String, Integer> properties = new Hashtable<>(); properties.put("port", listenPort); bundleContext.registerService(ServerSocketChannel.class, boundChannel, properties); } - ServerSocketChannel bindChannel(ConnectorConfig channelInfo) throws IOException { + static ServerSocketChannel bindChannel(ConnectorConfig channelInfo) throws IOException { ServerSocketChannel serverChannel = ServerSocketChannel.open(); InetSocketAddress bindAddress = new InetSocketAddress(channelInfo.listenPort()); serverChannel.socket().bind(bindAddress, channelInfo.acceptQueueSize()); @@ -117,7 +58,7 @@ public class StandaloneContainerActivator implements BundleActivator { @Override public void stop(BundleContext bundleContext) throws Exception { } - Container getContainer(Module... modules) { + static Container getContainer(Module... modules) { Module activatorModule = new ActivatorModule(); List<Module> allModules = new ArrayList<>(); allModules.addAll(Arrays.asList(modules)); @@ -127,7 +68,7 @@ public class StandaloneContainerActivator implements BundleActivator { return app.container(); } - List<ConnectorConfig> getConnectorConfigs(Container container) { + static List<ConnectorConfig> getConnectorConfigs(Container container) { Http http = container.getHttp(); return (http == null) ? diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java index 48c882c78db..8d413ade0f0 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.standalone; -import com.google.inject.Binder; import com.google.inject.Module; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.http.ConnectorConfig; @@ -18,6 +17,7 @@ import java.nio.file.Path; import java.util.List; import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.collection.IsEmptyCollection.empty; @@ -29,7 +29,7 @@ import static org.junit.Assert.assertThat; */ public class StandaloneContainerActivatorTest { - private String getJdiscXml(String contents) throws ParserConfigurationException, IOException, SAXException { + private static String getJdiscXml(String contents) throws ParserConfigurationException, IOException, SAXException { return "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + "<services>\n" + " <jdisc version=\"1.0\" jetty=\"true\">\n" + @@ -38,7 +38,7 @@ public class StandaloneContainerActivatorTest { "</services>"; } - private void writeApplicationPackage(String servicesXml, Path tmpDir) throws IOException { + private static void writeApplicationPackage(String servicesXml, Path tmpDir) throws IOException { FileWriter fw = new FileWriter(tmpDir.resolve("services.xml").toFile(), false); fw.write(servicesXml); fw.close(); @@ -50,16 +50,16 @@ public class StandaloneContainerActivatorTest { try { writeApplicationPackage(getJdiscXml(""), applicationDir); StandaloneContainerActivator activator = new StandaloneContainerActivator(); - Container container = activator.getContainer(newAppDirBinding(applicationDir)); + Container container = StandaloneContainerActivator.getContainer(newAppDirBinding(applicationDir)); List<Integer> ports = getPorts(activator, container); - assertThat(ports, is(asList(Defaults.getDefaults().vespaWebServicePort()))); + assertThat(ports, is(singletonList(Defaults.getDefaults().vespaWebServicePort()))); } finally { IOUtils.recursiveDeleteDir(applicationDir.toFile()); } } - private List<Integer> getPorts(StandaloneContainerActivator activator, Container container) { - return activator.getConnectorConfigs(container).stream(). + private static List<Integer> getPorts(StandaloneContainerActivator activator, Container container) { + return StandaloneContainerActivator.getConnectorConfigs(container).stream(). map(ConnectorConfig::listenPort). collect(toList()); } @@ -70,7 +70,7 @@ public class StandaloneContainerActivatorTest { try { writeApplicationPackage(getJdiscXml("<http/>"), applicationDir); StandaloneContainerActivator activator = new StandaloneContainerActivator(); - Container container = activator.getContainer(newAppDirBinding(applicationDir)); + Container container = StandaloneContainerActivator.getContainer(newAppDirBinding(applicationDir)); List<Integer> ports = getPorts(activator, container); assertThat(ports, empty()); } finally { @@ -90,7 +90,7 @@ public class StandaloneContainerActivatorTest { "</http>\n"; writeApplicationPackage(getJdiscXml(contents), applicationDir); StandaloneContainerActivator activator = new StandaloneContainerActivator(); - Container container = activator.getContainer(newAppDirBinding(applicationDir)); + Container container = StandaloneContainerActivator.getContainer(newAppDirBinding(applicationDir)); List<Integer> ports = getPorts(activator, container); assertThat(ports, is(asList(123, 456, 789))); } finally { @@ -98,15 +98,10 @@ public class StandaloneContainerActivatorTest { } } - private Module newAppDirBinding(final Path applicationDir) { - return new Module() { - @Override - public void configure(Binder binder) { - binder.bind(Path.class) - .annotatedWith(StandaloneContainerApplication.applicationPathName()) - .toInstance(applicationDir); - } - }; + private static Module newAppDirBinding(final Path applicationDir) { + return binder -> binder.bind(Path.class) + .annotatedWith(StandaloneContainerApplication.applicationPathName()) + .toInstance(applicationDir); } } diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index d1a54c04359..b9e33ea8d26 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -4,18 +4,23 @@ #include <iomanip> #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/storage/distributor/pending_bucket_space_db_transition.h> +#include <vespa/storage/distributor/outdated_nodes_map.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storage/storageutil/distributorstatecache.h> #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/distributor/simpleclusterinformation.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/vespalib/text/stringtokenizer.h> using namespace storage::api; using namespace storage::lib; using document::test::makeDocumentBucket; +using document::test::makeBucketSpace; namespace storage { namespace distributor { @@ -141,19 +146,21 @@ protected: void adding_diverging_replica_to_existing_trusted_does_not_remove_trusted(); void batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted(); + auto &defaultDistributorBucketSpace() { return getBucketSpaceRepo().get(makeBucketSpace()); } + bool bucketExistsThatHasNode(int bucketCount, uint16_t node) const; ClusterInformation::CSP createClusterInfo(const std::string& clusterState) { ClusterInformation::CSP clusterInfo( new SimpleClusterInformation( getBucketDBUpdater().getDistributorComponent().getIndex(), - getBucketDBUpdater().getDistributorComponent().getDistribution(), lib::ClusterState(clusterState), "ui")); return clusterInfo; } public: + using OutdatedNodesMap = dbtransition::OutdatedNodesMap; void setUp() override { createLinks(); }; @@ -181,8 +188,7 @@ public: } std::vector<uint16_t> nodes; - getBucketDBUpdater().getDistributorComponent() - .getDistribution().getIdealNodes( + defaultDistributorBucketSpace().getDistribution().getIdealNodes( lib::NodeType::STORAGE, state, document::BucketId(16, i), @@ -243,7 +249,7 @@ public: } std::vector<uint16_t> nodes; - getBucketDBUpdater().getDistributorComponent().getDistribution().getIdealNodes( + defaultDistributorBucketSpace().getDistribution().getIdealNodes( lib::NodeType::STORAGE, state, document::BucketId(id), @@ -536,9 +542,9 @@ public: ClusterInformation::CSP clusterInfo( owner.createClusterInfo(oldClusterState)); - std::unordered_set<uint16_t> outdatedNodes; + OutdatedNodesMap outdatedNodesMap; state = PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, cmd, outdatedNodes, + clock, clusterInfo, sender, owner.getBucketSpaceRepo(), cmd, outdatedNodesMap, api::Timestamp(1)); } @@ -549,9 +555,8 @@ public: ClusterInformation::CSP clusterInfo( owner.createClusterInfo(oldClusterState)); - std::unordered_set<uint16_t> outdatedNodes; state = PendingClusterState::createForDistributionChange( - clock, clusterInfo, sender, api::Timestamp(1)); + clock, clusterInfo, sender, owner.getBucketSpaceRepo(), api::Timestamp(1)); } }; @@ -582,7 +587,7 @@ BucketDBUpdaterTest::testNormalUsage() // Ensure distribution hash is set correctly CPPUNIT_ASSERT_EQUAL( - getBucketDBUpdater().getDistributorComponent().getDistribution() + defaultDistributorBucketSpace().getDistribution() .getNodeGraph().getDistributionConfigHash(), dynamic_cast<const RequestBucketInfoCommand&>( *_sender.commands[0]).getDistributionHash()); @@ -876,7 +881,7 @@ BucketDBUpdaterTest::testInitializingWhileRecheck() CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - getBucketDBUpdater().recheckBucketInfo(1, document::BucketId(16, 3)); + getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); for (int i=0; i<2; i++) { fakeBucketReply(systemState, @@ -913,8 +918,7 @@ BucketDBUpdaterTest::testBitChange() int cnt=0; for (int i=0; cnt < 2; i++) { - lib::Distribution distribution = getBucketDBUpdater().getDistributorComponent() - .getDistribution(); + lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution(); std::vector<uint16_t> distributors; if (distribution.getIdealDistributorNode( lib::ClusterState("redundancy:1 bits:14 storage:1 distributor:2"), @@ -1006,7 +1010,7 @@ BucketDBUpdaterTest::testRecheckNodeWithFailure() _sender.clear(); - getBucketDBUpdater().recheckBucketInfo(1, document::BucketId(16, 3)); + getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); @@ -1056,7 +1060,7 @@ BucketDBUpdaterTest::testRecheckNode() _sender.clear(); - getBucketDBUpdater().recheckBucketInfo(1, document::BucketId(16, 3)); + getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); @@ -1475,7 +1479,7 @@ BucketDBUpdaterTest::getSentNodesDistributionChanged( ClusterInformation::CSP clusterInfo(createClusterInfo(oldClusterState)); std::unique_ptr<PendingClusterState> state( PendingClusterState::createForDistributionChange( - clock, clusterInfo, sender, api::Timestamp(1))); + clock, clusterInfo, sender, getBucketSpaceRepo(), api::Timestamp(1))); sortSentMessagesByIndex(sender); @@ -1637,10 +1641,10 @@ BucketDBUpdaterTest::testPendingClusterStateReceive() framework::defaultimplementation::FakeClock clock; ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d")); - std::unordered_set<uint16_t> outdatedNodes; + OutdatedNodesMap outdatedNodesMap; std::unique_ptr<PendingClusterState> state( PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, cmd, outdatedNodes, + clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap, api::Timestamp(1))); CPPUNIT_ASSERT_EQUAL(3, (int)sender.commands.size()); @@ -1668,7 +1672,8 @@ BucketDBUpdaterTest::testPendingClusterStateReceive() state->done()); } - CPPUNIT_ASSERT_EQUAL(3, (int)state->results().size()); + auto &pendingTransition = state->getPendingBucketSpaceDbTransition(makeBucketSpace()); + CPPUNIT_ASSERT_EQUAL(3, (int)pendingTransition.results().size()); } void @@ -1721,13 +1726,14 @@ parseInputData(const std::string& data, uint16_t node = atoi(tok2[0].c_str()); state.setNodeReplied(node); + auto &pendingTransition = state.getPendingBucketSpaceDbTransition(makeBucketSpace()); vespalib::StringTokenizer tok3(tok2[1], ","); for (uint32_t j = 0; j < tok3.size(); j++) { if (includeBucketInfo) { vespalib::StringTokenizer tok4(tok3[j], "/"); - state.addNodeInfo( + pendingTransition.addNodeInfo( document::BucketId(16, atoi(tok4[0].c_str())), BucketCopy( timestamp, @@ -1739,7 +1745,7 @@ parseInputData(const std::string& data, atoi(tok4[2].c_str()), atoi(tok4[3].c_str())))); } else { - state.addNodeInfo( + pendingTransition.addNodeInfo( document::BucketId(16, atoi(tok3[j].c_str())), BucketCopy(timestamp, node, @@ -1793,7 +1799,7 @@ BucketDBUpdaterTest::mergeBucketLists( framework::MilliSecTimer timer(clock); MessageSenderStub sender; - std::unordered_set<uint16_t> outdatedNodes; + OutdatedNodesMap outdatedNodesMap; { auto cmd(std::make_shared<api::SetSystemStateCommand>(oldState)); @@ -1803,11 +1809,11 @@ BucketDBUpdaterTest::mergeBucketLists( ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d")); std::unique_ptr<PendingClusterState> state( PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, cmd, outdatedNodes, + clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap, beforeTime)); parseInputData(existingData, beforeTime, *state, includeBucketInfo); - state->mergeInto(getBucketDBUpdater().getDistributorComponent().getBucketDatabase()); + state->mergeIntoBucketDatabases(); } BucketDumper dumper_tmp(true); @@ -1822,19 +1828,17 @@ BucketDBUpdaterTest::mergeBucketLists( ClusterInformation::CSP clusterInfo(createClusterInfo(oldState.toString())); std::unique_ptr<PendingClusterState> state( PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, cmd, outdatedNodes, + clock, clusterInfo, sender, getBucketSpaceRepo(), cmd, outdatedNodesMap, afterTime)); parseInputData(newData, afterTime, *state, includeBucketInfo); - state->mergeInto(getBucketDBUpdater().getDistributorComponent() - .getBucketDatabase()); + state->mergeIntoBucketDatabases(); } BucketDumper dumper(includeBucketInfo); - getBucketDBUpdater().getDistributorComponent() - .getBucketDatabase().forEach(dumper); - getBucketDBUpdater().getDistributorComponent() - .getBucketDatabase().clear(); + auto &bucketDb(defaultDistributorBucketSpace().getBucketDatabase()); + bucketDb.forEach(dumper); + bucketDb.clear(); return dumper.ost.str(); } @@ -1949,7 +1953,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInCurrentState() } _sender.clear(); - getBucketDBUpdater().recheckBucketInfo(0, bucket); + getBucketDBUpdater().recheckBucketInfo(0, makeDocumentBucket(bucket)); CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); std::shared_ptr<api::RequestBucketInfoCommand> rbi( @@ -1981,7 +1985,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInPendingState() } _sender.clear(); - getBucketDBUpdater().recheckBucketInfo(0, bucket); + getBucketDBUpdater().recheckBucketInfo(0, makeDocumentBucket(bucket)); CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); std::shared_ptr<api::RequestBucketInfoCommand> rbi( @@ -1994,7 +1998,7 @@ BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInPendingState() CPPUNIT_ASSERT(getBucketDBUpdater().getDistributorComponent() .ownsBucketInCurrentState(makeDocumentBucket(bucket))); CPPUNIT_ASSERT(!getBucketDBUpdater() - .checkOwnershipInPendingState(bucket).isOwned()); + .checkOwnershipInPendingState(makeDocumentBucket(bucket)).isOwned()); sendFakeReplyForSingleBucketRequest(*rbi); diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index cbc78157911..1640af0f871 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -556,7 +556,7 @@ Distributor_Test::testNoDbResurrectionForBucketNotOwnedInPendingState() document::BucketId nonOwnedBucket(16, 3); CPPUNIT_ASSERT(!getBucketDBUpdater() - .checkOwnershipInPendingState(nonOwnedBucket).isOwned()); + .checkOwnershipInPendingState(makeDocumentBucket(nonOwnedBucket)).isOwned()); CPPUNIT_ASSERT(!getBucketDBUpdater().getDistributorComponent() .checkOwnershipInPendingAndCurrentState(makeDocumentBucket(nonOwnedBucket)) .isOwned()); diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index 5deb31f8579..6f5abc02512 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distributortestutil.h" #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/config-stor-distribution.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/document/test/make_document_bucket.h> @@ -358,6 +359,16 @@ DistributorTestUtil::getBucketDatabase() const { return _distributor->getDefaultBucketSpace().getBucketDatabase(); } +DistributorBucketSpaceRepo & +DistributorTestUtil::getBucketSpaceRepo() { + return _distributor->getBucketSpaceRepo(); +} + +const DistributorBucketSpaceRepo & +DistributorTestUtil::getBucketSpaceRepo() const { + return _distributor->getBucketSpaceRepo(); +} + const lib::Distribution& DistributorTestUtil::getDistribution() const { return _distributor->getDefaultBucketSpace().getDistribution(); diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h index 4f09c11ac03..19da0483165 100644 --- a/storage/src/tests/distributor/distributortestutil.h +++ b/storage/src/tests/distributor/distributortestutil.h @@ -20,6 +20,7 @@ namespace distributor { class BucketDBUpdater; class Distributor; class DistributorBucketSpace; +class DistributorBucketSpaceRepo; class IdealStateManager; class ExternalOperationHandler; class Operation; @@ -125,6 +126,8 @@ public: DistributorBucketSpace &getDistributorBucketSpace(); BucketDatabase& getBucketDatabase(); const BucketDatabase& getBucketDatabase() const; + DistributorBucketSpaceRepo &getBucketSpaceRepo(); + const DistributorBucketSpaceRepo &getBucketSpaceRepo() const; const lib::Distribution& getDistribution() const; // "End to end" distribution change trigger, which will invoke the bucket diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index 83e004f59fe..1fea6e47656 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -3,16 +3,22 @@ #include <vespa/config/helper/configgetter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <tests/distributor/distributortestutil.h> #include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/operationtargetresolverimpl.h> #include <vespa/storage/distributor/externaloperationhandler.h> #include <vespa/config/helper/configgetter.hpp> using document::BucketId; +using document::test::makeBucketSpace; +using document::test::makeDocumentBucket; namespace storage { namespace distributor { @@ -112,19 +118,19 @@ namespace { BucketInstanceList result(_test.getInstances(_id, true)); BucketInstanceList all(_test.getInstances(_id, false)); _asserter.assertEqualMsg( - all.toString(), _expected, result.createTargets()); + all.toString(), _expected, result.createTargets(makeBucketSpace())); delete _asserters.back(); _asserters.pop_back(); } TestTargets& sendsTo(const BucketId& id, uint16_t node) { _expected.push_back(OperationTarget( - id, lib::Node(lib::NodeType::STORAGE, node), false)); + makeDocumentBucket(id), lib::Node(lib::NodeType::STORAGE, node), false)); return *this; } TestTargets& createsAt(const BucketId& id, uint16_t node) { _expected.push_back(OperationTarget( - id, lib::Node(lib::NodeType::STORAGE, node), true)); + makeDocumentBucket(id), lib::Node(lib::NodeType::STORAGE, node), true)); return *this; } @@ -144,11 +150,14 @@ OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy) { lib::IdealNodeCalculatorImpl idealNodeCalc; - idealNodeCalc.setDistribution(getExternalOperationHandler().getDistribution()); + auto &bucketSpaceRepo(getExternalOperationHandler().getBucketSpaceRepo()); + auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); + idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); idealNodeCalc.setClusterState(getExternalOperationHandler().getClusterState()); OperationTargetResolverImpl resolver( - getExternalOperationHandler().getBucketDatabase(), idealNodeCalc, 16, - getExternalOperationHandler().getDistribution().getRedundancy()); + distributorBucketSpace.getBucketDatabase(), idealNodeCalc, 16, + distributorBucketSpace.getDistribution().getRedundancy(), + makeBucketSpace()); if (stripToRedundancy) { return resolver.getInstances(OperationTargetResolver::PUT, id); } else { @@ -174,11 +183,13 @@ OperationTargetResolverTest::testMultipleNodes() { setupDistributor(1, 2, "storage:2 distributor:1"); + auto &bucketSpaceRepo(getExternalOperationHandler().getBucketSpaceRepo()); + auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); for (int i = 0; i < 100; ++i) { addNodesToBucketDB(BucketId(16, i), "0=0,1=0"); lib::IdealNodeCalculatorImpl idealNodeCalc; - idealNodeCalc.setDistribution(getExternalOperationHandler().getDistribution()); + idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); idealNodeCalc.setClusterState(getExternalOperationHandler().getClusterState()); lib::IdealNodeList idealNodes( idealNodeCalc.getIdealStorageNodes(BucketId(16, i))); diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index a46419b71a4..66a2d3efa6c 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> @@ -11,11 +14,13 @@ namespace storage::distributor { using document::BucketId; +using document::test::makeBucketSpace; typedef MaintenancePriority Priority; class SimpleMaintenanceScannerTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SimpleMaintenanceScannerTest); CPPUNIT_TEST(testPrioritizeSingleBucket); + CPPUNIT_TEST(testPrioritizeSingleBucketAltBucketSpace); CPPUNIT_TEST(testPrioritizeMultipleBuckets); CPPUNIT_TEST(testPendingMaintenanceOperationStatistics); CPPUNIT_TEST(perNodeMaintenanceStatsAreTracked); @@ -27,10 +32,11 @@ class SimpleMaintenanceScannerTest : public CppUnit::TestFixture { std::string dumpPriorityDbToString(const BucketPriorityDatabase&) const; std::unique_ptr<MockMaintenancePriorityGenerator> _priorityGenerator; - std::unique_ptr<MapBucketDatabase> _bucketDb; + std::unique_ptr<DistributorBucketSpaceRepo> _bucketSpaceRepo; std::unique_ptr<SimpleBucketPriorityDatabase> _priorityDb; std::unique_ptr<SimpleMaintenanceScanner> _scanner; + void addBucketToDb(document::BucketSpace bucketSpace, int bucketNum); void addBucketToDb(int bucketNum); bool scanEntireDatabase(int expected); @@ -39,6 +45,7 @@ class SimpleMaintenanceScannerTest : public CppUnit::TestFixture { public: void testPrioritizeSingleBucket(); + void testPrioritizeSingleBucketAltBucketSpace(); void testPrioritizeMultipleBuckets(); void testPendingMaintenanceOperationStatistics(); void perNodeMaintenanceStatsAreTracked(); @@ -53,16 +60,23 @@ void SimpleMaintenanceScannerTest::setUp() { _priorityGenerator.reset(new MockMaintenancePriorityGenerator()); - _bucketDb.reset(new MapBucketDatabase()); + _bucketSpaceRepo = std::make_unique<DistributorBucketSpaceRepo>(); _priorityDb.reset(new SimpleBucketPriorityDatabase()); - _scanner.reset(new SimpleMaintenanceScanner(*_priorityDb, *_priorityGenerator, *_bucketDb)); + _scanner.reset(new SimpleMaintenanceScanner(*_priorityDb, *_priorityGenerator, *_bucketSpaceRepo)); } void -SimpleMaintenanceScannerTest::addBucketToDb(int bucketNum) +SimpleMaintenanceScannerTest::addBucketToDb(document::BucketSpace bucketSpace, int bucketNum) { BucketDatabase::Entry entry(BucketId(16, bucketNum), BucketInfo()); - _bucketDb->update(entry); + auto &bucketDb(_bucketSpaceRepo->get(bucketSpace).getBucketDatabase()); + bucketDb.update(entry); +} + +void +SimpleMaintenanceScannerTest::addBucketToDb(int bucketNum) +{ + addBucketToDb(makeBucketSpace(), bucketNum); } std::string @@ -80,7 +94,27 @@ SimpleMaintenanceScannerTest::testPrioritizeSingleBucket() addBucketToDb(1); std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"); - CPPUNIT_ASSERT(!_scanner->scanNext().isDone()); + auto scanResult = _scanner->scanNext(); + CPPUNIT_ASSERT(!scanResult.isDone()); + CPPUNIT_ASSERT_EQUAL(makeBucketSpace().getId(), scanResult.getBucketSpace().getId()); + CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); + + CPPUNIT_ASSERT(_scanner->scanNext().isDone()); + CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); +} + +void +SimpleMaintenanceScannerTest::testPrioritizeSingleBucketAltBucketSpace() +{ + document::BucketSpace bucketSpace(4); + _bucketSpaceRepo->add(bucketSpace, std::make_unique<DistributorBucketSpace>()); + _scanner->reset(); + addBucketToDb(bucketSpace, 1); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000004), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"); + + auto scanResult = _scanner->scanNext(); + CPPUNIT_ASSERT(!scanResult.isDone()); + CPPUNIT_ASSERT_EQUAL(bucketSpace.getId(), scanResult.getBucketSpace().getId()); CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); CPPUNIT_ASSERT(_scanner->scanNext().isDone()); diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 29c922248e7..306f92cdd6a 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -13,6 +13,8 @@ #include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> #include <vespa/storage/distributor/operations/idealstate/splitoperation.h> #include <vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storage/storageutil/utils.h> #include <tests/distributor/distributortestutil.h> @@ -20,8 +22,12 @@ #include <vespa/storageapi/message/state.h> #include <vespa/config-stor-distribution.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/test/make_document_bucket.h> using namespace std::literals::string_literals; +using document::test::makeBucketSpace; +using document::test::makeDocumentBucket; namespace storage { namespace distributor { @@ -105,8 +111,9 @@ struct StateCheckersTest : public CppUnit::TestFixture, void assertCurrentIdealState(const document::BucketId& bucket, const std::vector<uint16_t> expected) { + auto &distributorBucketSpace(getIdealStateManager().getBucketSpaceRepo().get(makeBucketSpace())); std::vector<uint16_t> idealNodes( - getIdealStateManager().getDistributorComponent() + distributorBucketSpace .getDistribution().getIdealStorageNodes( getIdealStateManager().getDistributorComponent() .getClusterState(), @@ -128,17 +135,17 @@ struct StateCheckersTest : public CppUnit::TestFixture, std::ostringstream ost; c.siblingBucket = getIdealStateManager().getDistributorComponent() - .getSibling(c.bucketId); + .getSibling(c.getBucketId()); std::vector<BucketDatabase::Entry> entries; - getBucketDatabase().getAll(c.bucketId, entries); + getBucketDatabase().getAll(c.getBucketId(), entries); c.siblingEntry = getBucketDatabase().get(c.siblingBucket); c.entries = entries; for (uint32_t j = 0; j < entries.size(); ++j) { // Run checking only on this bucketid, but include all buckets // owned by it or owners of it, so we can detect inconsistent split. - if (entries[j].getBucketId() == c.bucketId) { + if (entries[j].getBucketId() == c.getBucketId()) { c.entry = entries[j]; StateChecker::Result result(checker.check(c)); @@ -263,7 +270,7 @@ struct StateCheckersTest : public CppUnit::TestFixture, lib::ClusterState(params._clusterState)); NodeMaintenanceStatsTracker statsTracker; StateChecker::Context c( - getExternalOperationHandler(), statsTracker, bid); + getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); std::string result = testStateChecker( checker, c, false, *params._blockerMessage, params._includeMessagePriority, @@ -361,7 +368,7 @@ std::string StateCheckersTest::testSplit(uint32_t splitCount, SplitBucketStateChecker checker; NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); getConfig().setSplitSize(splitSize); getConfig().setSplitCount(splitCount); getConfig().setMinimalBucketSplit(minSplitBits); @@ -465,7 +472,7 @@ StateCheckersTest::testInconsistentSplit(const document::BucketId& bid, { SplitInconsistentStateChecker checker; NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); return testStateChecker(checker, c, true, PendingMessage(), includePriority); } @@ -533,7 +540,7 @@ StateCheckersTest::testJoin(uint32_t joinCount, getConfig().setMinimalBucketSplit(minSplitBits); NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); return testStateChecker(checker, c, true, blocker, includePriority); } @@ -789,7 +796,7 @@ StateCheckersTest::testSynchronizeAndMove(const std::string& bucketInfo, _distributor->enableClusterState(lib::ClusterState(clusterState)); NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); return testStateChecker(checker, c, false, blocker, includePriority); } @@ -820,7 +827,7 @@ StateCheckersTest::testSynchronizeAndMove() runAndVerify<SynchronizeAndMoveStateChecker>( CheckerParams() .expect("[Moving bucket to ideal node 3] " - "(scheduling pri VERY_LOW)") + "(scheduling pri LOW)") .bucketInfo("0=1,1=1,2=1") .clusterState("distributor:1 storage:4") .includeSchedulingPriority(true)); @@ -837,7 +844,7 @@ StateCheckersTest::testSynchronizeAndMove() CheckerParams() .expect("[Moving bucket to ideal node 1]" "[Moving bucket to ideal node 3] (pri 165) " - "(scheduling pri VERY_LOW)") + "(scheduling pri LOW)") .clusterState("distributor:1 storage:5") .bucketInfo("0=1,4=1,5=1") .includeMessagePriority(true) @@ -984,7 +991,7 @@ StateCheckersTest::testDeleteExtraCopies( } DeleteExtraCopiesStateChecker checker; NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); return testStateChecker(checker, c, false, blocker, includePriority); } @@ -995,8 +1002,9 @@ StateCheckersTest::testDeleteExtraCopies() setupDistributor(2, 100, "distributor:1 storage:4"); { + auto &distributorBucketSpace(getIdealStateManager().getBucketSpaceRepo().get(makeBucketSpace())); std::vector<uint16_t> idealNodes( - getIdealStateManager().getDistributorComponent() + distributorBucketSpace .getDistribution().getIdealStorageNodes( getIdealStateManager().getDistributorComponent().getClusterState(), document::BucketId(17, 0), @@ -1133,7 +1141,7 @@ std::string StateCheckersTest::testBucketState( BucketStateStateChecker checker; NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); return testStateChecker(checker, c, false, PendingMessage(), includePriority); } @@ -1332,7 +1340,7 @@ std::string StateCheckersTest::testBucketStatePerGroup( BucketStateStateChecker checker; NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket(bid)); return testStateChecker(checker, c, false, PendingMessage(), includePriority); } @@ -1474,8 +1482,8 @@ std::string StateCheckersTest::testGarbageCollection( getConfig().setGarbageCollection("music", checkInterval); getConfig().setLastGarbageCollectionChangeTime(lastChangeTime); NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, - e.getBucketId()); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, + makeDocumentBucket(e.getBucketId())); getClock().setAbsoluteTimeInSeconds(nowTimestamp); return testStateChecker(checker, c, false, PendingMessage(), includePriority, includeSchedulingPri); @@ -1533,7 +1541,7 @@ StateCheckersTest::testGarbageCollection() void StateCheckersTest::gc_ops_are_prioritized_with_low_priority_category() { CPPUNIT_ASSERT_EQUAL( std::string("[Needs garbage collection: Last check at 3, current time 4000, " - "configured interval 300] (scheduling pri LOW)"), + "configured interval 300] (scheduling pri VERY_LOW)"), testGarbageCollection(3, 4000, 300, 1, false, true)); } @@ -1561,8 +1569,8 @@ StateCheckersTest::gcInhibitedWhenIdealNodeInMaintenance() getConfig().setGarbageCollection("music", 3600); getConfig().setLastGarbageCollectionChangeTime(0); NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, - bucket); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, + makeDocumentBucket(bucket)); getClock().setAbsoluteTimeInSeconds(4000); // Would normally (in a non-maintenance case) trigger GC due to having // overshot the GC check cycle. @@ -1727,7 +1735,7 @@ StateCheckersTest::contextPopulatesIdealStateContainers() setupDistributor(2, 100, "distributor:1 storage:4"); NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(getExternalOperationHandler(), statsTracker, {17, 0}); + StateChecker::Context c(getExternalOperationHandler(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket({17, 0})); CPPUNIT_ASSERT_EQUAL((std::vector<uint16_t>{1, 3}), c.idealState); CPPUNIT_ASSERT_EQUAL(size_t(2), c.unorderedIdealState.size()); @@ -1772,7 +1780,7 @@ public: // NOTE: resets the bucket database! void runFor(const document::BucketId& bid) { Checker checker; - StateChecker::Context c(_fixture.getExternalOperationHandler(), _statsTracker, bid); + StateChecker::Context c(_fixture.getExternalOperationHandler(), _fixture.getDistributorBucketSpace(), _statsTracker, makeDocumentBucket(bid)); _result = _fixture.testStateChecker( checker, c, false, StateCheckersTest::PendingMessage(), false); } diff --git a/storage/src/tests/distributor/statoperationtest.cpp b/storage/src/tests/distributor/statoperationtest.cpp index 9ae8fc2fa4a..b010c5f6b79 100644 --- a/storage/src/tests/distributor/statoperationtest.cpp +++ b/storage/src/tests/distributor/statoperationtest.cpp @@ -8,6 +8,7 @@ #include <vespa/storage/distributor/operations/external/statbucketoperation.h> #include <vespa/storage/distributor/operations/external/statbucketlistoperation.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> using document::test::makeDocumentBucket; diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 1744bfb6a79..386be60d88c 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -1,20 +1,27 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/base/testdocrepo.h> #include <cppunit/extensions/HelperMacros.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/document/base/testdocrepo.h> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/select/parser.h> +#include <vespa/document/test/make_document_bucket.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/storage/common/bucket_resolver.h> #include <vespa/storage/storageserver/documentapiconverter.h> #include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/messagebus/emptyreply.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/config/subscription/configuri.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/document/test/make_document_bucket.h> +using document::Bucket; +using document::BucketId; +using document::BucketSpace; using document::DataType; using document::DocIdString; using document::Document; @@ -25,25 +32,81 @@ using document::test::makeDocumentBucket; namespace storage { +const DocumentId defaultDocId("id:test:text/html::0"); +const BucketSpace defaultBucketSpace(5); +const vespalib::string defaultSpaceName("myspace"); +const Bucket defaultBucket(defaultBucketSpace, BucketId(0)); + +struct MockBucketResolver : public BucketResolver { + virtual Bucket bucketFromId(const DocumentId &documentId) const override { + if (documentId.getDocType() == "text/html") { + return defaultBucket; + } + return Bucket(BucketSpace(0), BucketId(0)); + } + virtual BucketSpace bucketSpaceFromName(const vespalib::string &bucketSpace) const override { + if (bucketSpace == defaultSpaceName) { + return defaultBucketSpace; + } + return BucketSpace(0); + } + virtual vespalib::string nameFromBucketSpace(const document::BucketSpace &bucketSpace) const override { + if (bucketSpace == defaultBucketSpace) { + return defaultSpaceName; + } + return ""; + } +}; + struct DocumentApiConverterTest : public CppUnit::TestFixture { + MockBucketResolver _bucketResolver; std::unique_ptr<DocumentApiConverter> _converter; const DocumentTypeRepo::SP _repo; const DataType& _html_type; DocumentApiConverterTest() - : _repo(new DocumentTypeRepo(readDocumenttypesConfig( + : _bucketResolver(), + _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig( TEST_PATH("config-doctypes.cfg")))), _html_type(*_repo->getDocumentType("text/html")) { } void setUp() override { - _converter.reset(new DocumentApiConverter("raw:")); + _converter.reset(new DocumentApiConverter("raw:", _bucketResolver)); }; + template <typename DerivedT, typename BaseT> + std::unique_ptr<DerivedT> dynamic_unique_ptr_cast(std::unique_ptr<BaseT> base) { + auto derived = dynamic_cast<DerivedT*>(base.get()); + CPPUNIT_ASSERT(derived); + base.release(); + return std::unique_ptr<DerivedT>(derived); + } + + template <typename T> + std::unique_ptr<T> toStorageAPI(documentapi::DocumentMessage &msg) { + auto result = _converter->toStorageAPI(msg, _repo); + return dynamic_unique_ptr_cast<T>(std::move(result)); + } + + template <typename T> + std::unique_ptr<T> toStorageAPI(mbus::Reply &fromReply, + api::StorageCommand &fromCommand) { + auto result = _converter->toStorageAPI(static_cast<documentapi::DocumentReply&>(fromReply), fromCommand); + return dynamic_unique_ptr_cast<T>(std::move(result)); + } + + template <typename T> + std::unique_ptr<T> toDocumentAPI(api::StorageCommand &cmd) { + auto result = _converter->toDocumentAPI(cmd, _repo); + return dynamic_unique_ptr_cast<T>(std::move(result)); + } + void testPut(); void testForwardedPut(); + void testUpdate(); void testRemove(); void testGet(); void testCreateVisitor(); @@ -54,10 +117,14 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture void testVisitorInfo(); void testMultiOperation(); void testBatchDocumentUpdate(); + void testStatBucket(); + void testGetBucketList(); + void testRemoveLocation(); CPPUNIT_TEST_SUITE(DocumentApiConverterTest); CPPUNIT_TEST(testPut); CPPUNIT_TEST(testForwardedPut); + CPPUNIT_TEST(testUpdate); CPPUNIT_TEST(testRemove); CPPUNIT_TEST(testGet); CPPUNIT_TEST(testCreateVisitor); @@ -68,6 +135,9 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture CPPUNIT_TEST(testVisitorInfo); CPPUNIT_TEST(testMultiOperation); CPPUNIT_TEST(testBatchDocumentUpdate); + CPPUNIT_TEST(testStatBucket); + CPPUNIT_TEST(testGetBucketList); + CPPUNIT_TEST(testRemoveLocation); CPPUNIT_TEST_SUITE_END(); }; @@ -75,128 +145,126 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DocumentApiConverterTest); void DocumentApiConverterTest::testPut() { - Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + auto doc = std::make_shared<Document>(_html_type, defaultDocId); documentapi::PutDocumentMessage putmsg(doc); putmsg.setTimestamp(1234); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(putmsg, _repo); - api::PutCommand* pc = dynamic_cast<api::PutCommand*>(cmd.get()); - - CPPUNIT_ASSERT(pc); - CPPUNIT_ASSERT(pc->getDocument().get() == doc.get()); + auto cmd = toStorageAPI<api::PutCommand>(putmsg); + CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); + CPPUNIT_ASSERT(cmd->getDocument().get() == doc.get()); std::unique_ptr<mbus::Reply> reply = putmsg.createReply(); CPPUNIT_ASSERT(reply.get()); - std::unique_ptr<storage::api::StorageReply> rep = _converter->toStorageAPI( - static_cast<documentapi::DocumentReply&>(*reply), *cmd); - api::PutReply* pr = dynamic_cast<api::PutReply*>(rep.get()); - CPPUNIT_ASSERT(pr); - - std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(*pc, _repo); + toStorageAPI<api::PutReply>(*reply, *cmd); - documentapi::PutDocumentMessage* mbusput = dynamic_cast<documentapi::PutDocumentMessage*>(mbusmsg.get()); - CPPUNIT_ASSERT(mbusput); - CPPUNIT_ASSERT(mbusput->getDocumentSP().get() == doc.get()); - CPPUNIT_ASSERT(mbusput->getTimestamp() == 1234); -}; + auto mbusPut = toDocumentAPI<documentapi::PutDocumentMessage>(*cmd); + CPPUNIT_ASSERT(mbusPut->getDocumentSP().get() == doc.get()); + CPPUNIT_ASSERT(mbusPut->getTimestamp() == 1234); +} void DocumentApiConverterTest::testForwardedPut() { - Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + auto doc = std::make_shared<Document>(_html_type, DocumentId(DocIdString("test", "test"))); documentapi::PutDocumentMessage* putmsg = new documentapi::PutDocumentMessage(doc); std::unique_ptr<mbus::Reply> reply(((documentapi::DocumentMessage*)putmsg)->createReply()); reply->setMessage(std::unique_ptr<mbus::Message>(putmsg)); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(*putmsg, _repo); - ((storage::api::PutCommand*)cmd.get())->setTimestamp(1234); + auto cmd = toStorageAPI<api::PutCommand>(*putmsg); + cmd->setTimestamp(1234); - std::unique_ptr<storage::api::StorageReply> rep = cmd->makeReply(); - api::PutReply* pr = dynamic_cast<api::PutReply*>(rep.get()); - CPPUNIT_ASSERT(pr); + auto rep = dynamic_unique_ptr_cast<api::PutReply>(cmd->makeReply()); + _converter->transferReplyState(*rep, *reply); +} - _converter->transferReplyState(*pr, *reply); +void DocumentApiConverterTest::testUpdate() +{ + auto update = std::make_shared<document::DocumentUpdate>(_html_type, defaultDocId); + documentapi::UpdateDocumentMessage updateMsg(update); + updateMsg.setOldTimestamp(1234); + updateMsg.setNewTimestamp(5678); + + auto updateCmd = toStorageAPI<api::UpdateCommand>(updateMsg); + CPPUNIT_ASSERT_EQUAL(defaultBucket, updateCmd->getBucket()); + CPPUNIT_ASSERT_EQUAL(update.get(), updateCmd->getUpdate().get()); + CPPUNIT_ASSERT_EQUAL(api::Timestamp(1234), updateCmd->getOldTimestamp()); + CPPUNIT_ASSERT_EQUAL(api::Timestamp(5678), updateCmd->getTimestamp()); + + auto mbusReply = updateMsg.createReply(); + CPPUNIT_ASSERT(mbusReply.get()); + toStorageAPI<api::UpdateReply>(*mbusReply, *updateCmd); + + auto mbusUpdate = toDocumentAPI<documentapi::UpdateDocumentMessage>(*updateCmd); + CPPUNIT_ASSERT((&mbusUpdate->getDocumentUpdate()) == update.get()); + CPPUNIT_ASSERT_EQUAL(api::Timestamp(1234), mbusUpdate->getOldTimestamp()); + CPPUNIT_ASSERT_EQUAL(api::Timestamp(5678), mbusUpdate->getNewTimestamp()); } void DocumentApiConverterTest::testRemove() { - documentapi::RemoveDocumentMessage removemsg(document::DocumentId(document::DocIdString("test", "test"))); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(removemsg, _repo); - - api::RemoveCommand* rc = dynamic_cast<api::RemoveCommand*>(cmd.get()); - - CPPUNIT_ASSERT(rc); - CPPUNIT_ASSERT_EQUAL(document::DocumentId(document::DocIdString("test", "test")), rc->getDocumentId()); + documentapi::RemoveDocumentMessage removemsg(defaultDocId); + auto cmd = toStorageAPI<api::RemoveCommand>(removemsg); + CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); + CPPUNIT_ASSERT_EQUAL(defaultDocId, cmd->getDocumentId()); std::unique_ptr<mbus::Reply> reply = removemsg.createReply(); CPPUNIT_ASSERT(reply.get()); - std::unique_ptr<storage::api::StorageReply> rep = _converter->toStorageAPI( - static_cast<documentapi::DocumentReply&>(*reply), *cmd); - api::RemoveReply* pr = dynamic_cast<api::RemoveReply*>(rep.get()); - CPPUNIT_ASSERT(pr); - - std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(*rc, _repo); + toStorageAPI<api::RemoveReply>(*reply, *cmd); - documentapi::RemoveDocumentMessage* mbusremove = dynamic_cast<documentapi::RemoveDocumentMessage*>(mbusmsg.get()); - CPPUNIT_ASSERT(mbusremove); - CPPUNIT_ASSERT_EQUAL(document::DocumentId(document::DocIdString("test", "test")), mbusremove->getDocumentId()); -}; + auto mbusRemove = toDocumentAPI<documentapi::RemoveDocumentMessage>(*cmd); + CPPUNIT_ASSERT_EQUAL(defaultDocId, mbusRemove->getDocumentId()); +} void DocumentApiConverterTest::testGet() { - documentapi::GetDocumentMessage getmsg( - document::DocumentId(document::DocIdString("test", "test")), "foo bar"); - - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(getmsg, _repo); + documentapi::GetDocumentMessage getmsg(defaultDocId, "foo bar"); - api::GetCommand* rc = dynamic_cast<api::GetCommand*>(cmd.get()); - - CPPUNIT_ASSERT(rc); - CPPUNIT_ASSERT_EQUAL(document::DocumentId(document::DocIdString("test", "test")), rc->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo bar"), rc->getFieldSet()); -}; + auto cmd = toStorageAPI<api::GetCommand>(getmsg); + CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); + CPPUNIT_ASSERT_EQUAL(defaultDocId, cmd->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo bar"), cmd->getFieldSet()); +} void DocumentApiConverterTest::testCreateVisitor() { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); - + cv.setBucketSpace(defaultSpaceName); cv.setTimeRemaining(123456); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); - api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get()); - - CPPUNIT_ASSERT(pc); - CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), pc->getLibraryName()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), pc->getControlDestination()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), pc->getDataDestination()); - CPPUNIT_ASSERT_EQUAL(123456u, pc->getTimeout()); + + auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); + CPPUNIT_ASSERT_EQUAL(defaultBucketSpace, cmd->getBucket().getBucketSpace()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), cmd->getLibraryName()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), cmd->getInstanceId()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), cmd->getControlDestination()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), cmd->getDataDestination()); + CPPUNIT_ASSERT_EQUAL(123456u, cmd->getTimeout()); + + auto msg = toDocumentAPI<documentapi::CreateVisitorMessage>(*cmd); + CPPUNIT_ASSERT_EQUAL(defaultSpaceName, msg->getBucketSpace()); } void DocumentApiConverterTest::testCreateVisitorHighTimeout() { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); cv.setTimeRemaining((uint64_t)std::numeric_limits<uint32_t>::max() + 1); // Will be INT_MAX - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); - api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get()); - - CPPUNIT_ASSERT(pc); - CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), pc->getLibraryName()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), pc->getControlDestination()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), pc->getDataDestination()); - CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), pc->getTimeout()); + + auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); + CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), cmd->getLibraryName()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), cmd->getInstanceId()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), cmd->getControlDestination()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), cmd->getDataDestination()); + CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), cmd->getTimeout()); } void DocumentApiConverterTest::testCreateVisitorReplyNotReady() { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); - CPPUNIT_ASSERT(cmd.get()); - api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd); - api::CreateVisitorReply cvr(cvc); + + auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); + api::CreateVisitorReply cvr(*cmd); cvr.setResult(api::ReturnCode(api::ReturnCode::NOT_READY, "not ready")); std::unique_ptr<documentapi::CreateVisitorReply> reply( @@ -207,14 +275,12 @@ void DocumentApiConverterTest::testCreateVisitorReplyNotReady() CPPUNIT_ASSERT_EQUAL(document::BucketId(std::numeric_limits<int>::max()), reply->getLastBucket()); } - void DocumentApiConverterTest::testCreateVisitorReplyLastBucket() { documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); - CPPUNIT_ASSERT(cmd.get()); - api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd); - api::CreateVisitorReply cvr(cvc); + + auto cmd = toStorageAPI<api::CreateVisitorCommand>(cv); + api::CreateVisitorReply cvr(*cmd); cvr.setLastBucket(document::BucketId(123)); std::unique_ptr<documentapi::CreateVisitorReply> reply( dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release())); @@ -224,17 +290,12 @@ void DocumentApiConverterTest::testCreateVisitorReplyLastBucket() CPPUNIT_ASSERT_EQUAL(document::BucketId(123), reply->getLastBucket()); } - void DocumentApiConverterTest::testDestroyVisitor() { documentapi::DestroyVisitorMessage cv("myinstance"); - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); - - api::DestroyVisitorCommand* pc = dynamic_cast<api::DestroyVisitorCommand*>(cmd.get()); - - CPPUNIT_ASSERT(pc); - CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); + auto cmd = toStorageAPI<api::DestroyVisitorCommand>(cv); + CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), cmd->getInstanceId()); } void @@ -248,10 +309,7 @@ DocumentApiConverterTest::testVisitorInfo() vicmd.setBucketsCompleted(bucketsCompleted); - std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(vicmd, _repo); - - documentapi::VisitorInfoMessage* mbusvi = dynamic_cast<documentapi::VisitorInfoMessage*>(mbusmsg.get()); - CPPUNIT_ASSERT(mbusvi); + auto mbusvi = toDocumentAPI<documentapi::VisitorInfoMessage>(vicmd); CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), mbusvi->getFinishedBuckets()[0]); CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 2), mbusvi->getFinishedBuckets()[1]); CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 4), mbusvi->getFinishedBuckets()[2]); @@ -259,17 +317,13 @@ DocumentApiConverterTest::testVisitorInfo() std::unique_ptr<mbus::Reply> reply = mbusvi->createReply(); CPPUNIT_ASSERT(reply.get()); - std::unique_ptr<storage::api::StorageReply> rep = _converter->toStorageAPI( - static_cast<documentapi::DocumentReply&>(*reply), vicmd); - api::VisitorInfoReply* pr = dynamic_cast<api::VisitorInfoReply*>(rep.get()); - CPPUNIT_ASSERT(pr); + toStorageAPI<api::VisitorInfoReply>(*reply, vicmd); } void DocumentApiConverterTest::testMultiOperation() { - //create a document - Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + auto doc = std::make_shared<Document>(_html_type, DocumentId(DocIdString("test", "test"))); document::BucketIdFactory fac; document::BucketId bucketId = fac.getBucketId(doc->getId()); @@ -284,10 +338,7 @@ DocumentApiConverterTest::testMultiOperation() CPPUNIT_ASSERT(momsg.getBuffer().size() > 0); // Convert it to Storage API - std::unique_ptr<api::StorageCommand> stcmd = _converter->toStorageAPI(momsg, _repo); - - api::MultiOperationCommand* mocmd = dynamic_cast<api::MultiOperationCommand*>(stcmd.get()); - CPPUNIT_ASSERT(mocmd); + auto mocmd = toStorageAPI<api::MultiOperationCommand>(momsg); CPPUNIT_ASSERT(mocmd->getBuffer().size() > 0); // Get operations from Storage API message and check document @@ -296,7 +347,7 @@ DocumentApiConverterTest::testMultiOperation() CPPUNIT_ASSERT_EQUAL(*doc, *dynamic_cast<document::Document*>(list.begin()->getDocument().get())); // Create Storage API Reply - std::unique_ptr<api::MultiOperationReply> moreply = std::unique_ptr<api::MultiOperationReply>(new api::MultiOperationReply(*mocmd)); + auto moreply = std::make_unique<api::MultiOperationReply>(*mocmd); CPPUNIT_ASSERT(moreply.get()); // convert storage api reply to mbus reply..... @@ -308,9 +359,7 @@ DocumentApiConverterTest::testMultiOperation() mocmd.getOperations().addPut(*doc, 100); // Convert it to documentapi - std::unique_ptr<mbus::Message> mbmsg = _converter->toDocumentAPI(mocmd, _repo); - documentapi::MultiOperationMessage* momsg = dynamic_cast<documentapi::MultiOperationMessage*>(mbmsg.get()); - CPPUNIT_ASSERT(momsg); + auto momsg = toDocumentAPI<documentapi::MultiOperationMessage>(mocmd); // Get operations from Document API msg and check document const vdslib::DocumentList& list = momsg->getOperations(); @@ -322,11 +371,7 @@ DocumentApiConverterTest::testMultiOperation() CPPUNIT_ASSERT(moreply.get()); //Convert DocumentAPI reply to storageapi reply - std::unique_ptr<api::StorageReply> streply = - _converter->toStorageAPI(static_cast<documentapi::DocumentReply&>(*moreply), mocmd); - api::MultiOperationReply* mostreply = dynamic_cast<api::MultiOperationReply*>(streply.get()); - CPPUNIT_ASSERT(mostreply); - + toStorageAPI<api::MultiOperationReply>(*moreply, mocmd); } } @@ -337,19 +382,19 @@ DocumentApiConverterTest::testBatchDocumentUpdate() { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1")); - document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId)); + auto update = std::make_shared<document::DocumentUpdate>(_html_type, docId); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2")); - document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId)); + auto update = std::make_shared<document::DocumentUpdate>(_html_type, docId); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3")); - document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId)); + auto update = std::make_shared<document::DocumentUpdate>(_html_type, docId); updates.push_back(update); } @@ -358,9 +403,7 @@ DocumentApiConverterTest::testBatchDocumentUpdate() msg->addUpdate(updates[i]); } - std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(*msg, _repo); - api::BatchDocumentUpdateCommand* batchCmd = dynamic_cast<api::BatchDocumentUpdateCommand*>(cmd.get()); - CPPUNIT_ASSERT(batchCmd); + auto batchCmd = toStorageAPI<api::BatchDocumentUpdateCommand>(*msg); CPPUNIT_ASSERT_EQUAL(updates.size(), batchCmd->getUpdates().size()); for (std::size_t i = 0; i < updates.size(); ++i) { CPPUNIT_ASSERT_EQUAL(*updates[i], *batchCmd->getUpdates()[i]); @@ -384,4 +427,40 @@ DocumentApiConverterTest::testBatchDocumentUpdate() CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[2] == true); } +void +DocumentApiConverterTest::testStatBucket() +{ + documentapi::StatBucketMessage msg(BucketId(123), ""); + msg.setBucketSpace(defaultSpaceName); + + auto cmd = toStorageAPI<api::StatBucketCommand>(msg); + CPPUNIT_ASSERT_EQUAL(Bucket(defaultBucketSpace, BucketId(123)), cmd->getBucket()); + + auto mbusMsg = toDocumentAPI<documentapi::StatBucketMessage>(*cmd); + CPPUNIT_ASSERT_EQUAL(BucketId(123), mbusMsg->getBucketId()); + CPPUNIT_ASSERT_EQUAL(defaultSpaceName, mbusMsg->getBucketSpace()); +} + +void +DocumentApiConverterTest::testGetBucketList() +{ + documentapi::GetBucketListMessage msg(BucketId(123)); + msg.setBucketSpace(defaultSpaceName); + + auto cmd = toStorageAPI<api::GetBucketListCommand>(msg); + CPPUNIT_ASSERT_EQUAL(Bucket(defaultBucketSpace, BucketId(123)), cmd->getBucket()); +} + +void +DocumentApiConverterTest::testRemoveLocation() +{ + document::BucketIdFactory factory; + document::select::Parser parser(*_repo, factory); + documentapi::RemoveLocationMessage msg(factory, parser, "id.group == \"mygroup\""); + msg.setBucketSpace(defaultSpaceName); + + auto cmd = toStorageAPI<api::RemoveLocationCommand>(msg); + CPPUNIT_ASSERT_EQUAL(defaultBucket, cmd->getBucket()); +} + } diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 7d04714e9a8..3d469fc4252 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -313,7 +313,7 @@ MergeThrottlerTest::testChain() _servers[i]->setClusterState(lib::ClusterState("distributor:100 storage:100 version:123")); } - BucketId bid(14, 0x1337); + Bucket bucket(makeDocumentBucket(BucketId(14, 0x1337))); // Use different node permutations to ensure it works no matter which node is // set as the executor. More specifically, _all_ permutations. @@ -321,15 +321,11 @@ MergeThrottlerTest::testChain() uint16_t lastNodeIdx = _storageNodeCount - 1; uint16_t executorNode = indices[0]; - //std::cout << "\n----\n"; std::vector<MergeBucketCommand::Node> nodes; for (int i = 0; i < _storageNodeCount; ++i) { nodes.push_back(MergeBucketCommand::Node(indices[i], (i + executorNode) % 2 == 0)); - //std::cout << indices[i] << " "; } - //std::cout << "\n"; - std::shared_ptr<MergeBucketCommand> cmd( - new MergeBucketCommand(makeDocumentBucket(bid), nodes, UINT_MAX, 123)); + auto cmd = std::make_shared<MergeBucketCommand>(bucket, nodes, UINT_MAX, 123); cmd->setPriority(7); cmd->setTimeout(54321); StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); @@ -351,8 +347,6 @@ MergeThrottlerTest::testChain() _topLinks[i]->sendDown(fwd); _topLinks[i]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); - //std::cout << "fwd " << i << " -> " << i+1 << "\n"; - // Forwarded merge should not be sent down. Should not be necessary // to lock throttler here, since it should be sleeping like a champion CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[i]->getNumCommands()); @@ -363,7 +357,6 @@ MergeThrottlerTest::testChain() CPPUNIT_ASSERT_EQUAL(uint16_t(i + 1), fwd->getAddress()->getIndex()); CPPUNIT_ASSERT_EQUAL(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); { - //uint16_t chain[] = { 0 }; std::vector<uint16_t> chain; for (int j = 0; j <= i; ++j) { chain.push_back(j); @@ -416,10 +409,10 @@ MergeThrottlerTest::testChain() // The MergeBucketCommand that is kept in the executor node should // be the one from the node it initially got it from, NOT the one // from the last node, since the chain has looped - CPPUNIT_ASSERT(_throttlers[executorNode]->getActiveMerges().find(bid) + CPPUNIT_ASSERT(_throttlers[executorNode]->getActiveMerges().find(bucket) != _throttlers[executorNode]->getActiveMerges().end()); CPPUNIT_ASSERT_EQUAL(static_cast<StorageMessage*>(fwdToExec.get()), - _throttlers[executorNode]->getActiveMerges().find(bid)->second.getMergeCmd().get()); + _throttlers[executorNode]->getActiveMerges().find(bucket)->second.getMergeCmd().get()); } // Send reply up from persistence layer to simulate a completed @@ -440,7 +433,7 @@ MergeThrottlerTest::testChain() // Merge should not be removed yet from executor, since it's pending an unwind CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[executorNode]->getActiveMerges().size()); CPPUNIT_ASSERT_EQUAL(static_cast<StorageMessage*>(fwdToExec.get()), - _throttlers[executorNode]->getActiveMerges().find(bid)->second.getMergeCmd().get()); + _throttlers[executorNode]->getActiveMerges().find(bucket)->second.getMergeCmd().get()); } // MergeBucketReply waiting to be sent back to node 2. NOTE: we don't have any // transport context stuff set up here to perform the reply mapping, so we @@ -452,8 +445,6 @@ MergeThrottlerTest::testChain() // eg: 0 -> 2 -> 1 -> 0. Or: 2 -> 1 -> 0 if no cycle for (int i = (executorNode != lastNodeIdx ? _storageNodeCount - 1 : _storageNodeCount - 2); i >= 0; --i) { - //std::cout << "unwind " << i << "\n"; - _topLinks[i]->sendDown(unwind); _topLinks[i]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); @@ -469,7 +460,7 @@ MergeThrottlerTest::testChain() CPPUNIT_ASSERT_EQUAL(ReturnCode::OK, mbr.getResult().getResult()); CPPUNIT_ASSERT_EQUAL(vespalib::string("Great success! :D-|-<"), mbr.getResult().getMessage()); - CPPUNIT_ASSERT_EQUAL(bid, mbr.getBucketId()); + CPPUNIT_ASSERT_EQUAL(bucket, mbr.getBucket()); } while (std::next_permutation(indices, indices + _storageNodeCount)); diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 5e7cf4af046..00fa5c95c9b 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -104,14 +104,14 @@ namespace { DistributorStateCache _state; std::unordered_map<uint16_t, ResultArray>& _result; const document::BucketIdFactory& _factory; - std::shared_ptr<lib::Distribution> _storageDistribution; + std::shared_ptr<const lib::Distribution> _storageDistribution; public: DistributorInfoGatherer( const lib::ClusterState& systemState, std::unordered_map<uint16_t, ResultArray>& result, const document::BucketIdFactory& factory, - std::shared_ptr<lib::Distribution> distribution) + std::shared_ptr<const lib::Distribution> distribution) : _state(*distribution, systemState), _result(result), _factory(factory), @@ -513,7 +513,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac typedef std::shared_ptr<api::RequestBucketInfoCommand> RBISP; std::map<uint16_t, RBISP> requests; - lib::Distribution::SP distribution(_component.getDistribution()); + auto distribution(_component.getBucketSpaceRepo().get(bucketSpace).getDistribution()); lib::ClusterState::CSP clusterState( _component.getStateUpdater().getSystemState()); assert(clusterState.get()); diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index 03d94b27f0b..a4382ceb683 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -16,10 +16,12 @@ #include <map> #include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/bucket/bucketid.h> +#include <mutex> +#include <condition_variable> +#include <cassert> namespace storage { @@ -238,7 +240,8 @@ private: }; Map _map; - vespalib::Monitor _lock; + mutable std::mutex _lock; + std::condition_variable _cond; LockIdSet _lockedKeys; LockWaiters _lockWaiters; @@ -247,9 +250,9 @@ private: const char* clientId, bool haslock, bool& preExisted); void unlock(const key_type& key); bool findNextKey(key_type& key, mapped_type& val, const char* clientId, - vespalib::MonitorGuard& guard); + std::unique_lock<std::mutex> &guard); bool handleDecision(key_type& key, mapped_type& val, Decision decision); - void ackquireKey(const LockId & lid, vespalib::MonitorGuard & guard); + void acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard); /** * Process up to `chunkSize` bucket database entries from--and possibly @@ -304,7 +307,7 @@ private: void addAndLockResults(const std::vector<BucketId::Type> keys, const char* clientId, std::map<BucketId, WrappedEntry>& results, - vespalib::MonitorGuard& guard); + std::unique_lock<std::mutex> &guard); }; } // storage diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index f5d692139be..f370a792145 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -69,6 +69,7 @@ template<typename Map> LockableMap<Map>::LockableMap() : _map(), _lock(), + _cond(), _lockedKeys(), _lockWaiters() {} @@ -80,8 +81,8 @@ template<typename Map> bool LockableMap<Map>::operator==(const LockableMap<Map>& other) const { - vespalib::LockGuard guard(_lock); - vespalib::LockGuard guard2(other._lock); + std::lock_guard<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard2(other._lock); return (_map == other._map); } @@ -89,8 +90,8 @@ template<typename Map> bool LockableMap<Map>::operator<(const LockableMap<Map>& other) const { - vespalib::LockGuard guard(_lock); - vespalib::LockGuard guard2(other._lock); + std::lock_guard<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard2(other._lock); return (_map < other._map); } @@ -98,7 +99,7 @@ template<typename Map> typename Map::size_type LockableMap<Map>::size() const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _map.size(); } @@ -106,17 +107,16 @@ template<typename Map> typename Map::size_type LockableMap<Map>::getMemoryUsage() const { - vespalib::MonitorGuard guard(_lock); - return _map.getMemoryUsage() - + _lockedKeys.getMemoryUsage() - + sizeof(vespalib::Monitor); + std::lock_guard<std::mutex> guard(_lock); + return _map.getMemoryUsage() + _lockedKeys.getMemoryUsage() + + sizeof(std::mutex) + sizeof(std::condition_variable); } template<typename Map> bool LockableMap<Map>::empty() const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); return _map.empty(); } @@ -124,18 +124,18 @@ template<typename Map> void LockableMap<Map>::swap(LockableMap<Map>& other) { - vespalib::LockGuard guard(_lock); - vespalib::LockGuard guard2(other._lock); + std::lock_guard<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard2(other._lock); return _map.swap(other._map); } template<typename Map> -void LockableMap<Map>::ackquireKey(const LockId & lid, vespalib::MonitorGuard & guard) +void LockableMap<Map>::acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard) { if (_lockedKeys.exist(lid)) { typename LockWaiters::Key waitId(_lockWaiters.insert(lid)); while (_lockedKeys.exist(lid)) { - guard.wait(); + _cond.wait(guard); } _lockWaiters.erase(waitId); } @@ -148,8 +148,8 @@ LockableMap<Map>::get(const key_type& key, const char* clientId, bool lockIfNonExistingAndNotCreating) { LockId lid(key, clientId); - vespalib::MonitorGuard guard(_lock); - ackquireKey(lid, guard); + std::unique_lock<std::mutex> guard(_lock); + acquireKey(lid, guard); bool preExisted = false; typename Map::iterator it = _map.find(key, createIfNonExisting, preExisted); @@ -197,9 +197,9 @@ bool LockableMap<Map>::erase(const key_type& key, const char* clientId, bool haslock) { LockId lid(key, clientId); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (!haslock) { - ackquireKey(lid, guard); + acquireKey(lid, guard); } #ifdef ENABLE_BUCKET_OPERATION_LOGGING debug::logBucketDbErase(key, debug::TypeTag<mapped_type>()); @@ -213,9 +213,9 @@ LockableMap<Map>::insert(const key_type& key, const mapped_type& value, const char* clientId, bool haslock, bool& preExisted) { LockId lid(key, clientId); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (!haslock) { - ackquireKey(lid, guard); + acquireKey(lid, guard); } #ifdef ENABLE_BUCKET_OPERATION_LOGGING debug::logBucketDbInsert(key, value); @@ -227,7 +227,7 @@ template<typename Map> void LockableMap<Map>::clear() { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _map.clear(); } @@ -235,13 +235,13 @@ template<typename Map> bool LockableMap<Map>::findNextKey(key_type& key, mapped_type& val, const char* clientId, - vespalib::MonitorGuard& guard) + std::unique_lock<std::mutex> &guard) { // Wait for next value to unlock. typename Map::iterator it(_map.lower_bound(key)); while (it != _map.end() && _lockedKeys.exist(LockId(it->first, ""))) { typename LockWaiters::Key waitId(_lockWaiters.insert(LockId(it->first, clientId))); - guard.wait(); + _cond.wait(guard); _lockWaiters.erase(waitId); it = _map.lower_bound(key); } @@ -279,16 +279,16 @@ LockableMap<Map>::each(Functor& functor, const char* clientId, mapped_type val; Decision decision; { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (findNextKey(key, val, clientId, guard) || key > last) return; _lockedKeys.insert(LockId(key, clientId)); } try{ while (true) { decision = functor(const_cast<const key_type&>(key), val); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); if (handleDecision(key, val, decision)) return; ++key; if (findNextKey(key, val, clientId, guard) || key > last) return; @@ -297,9 +297,9 @@ LockableMap<Map>::each(Functor& functor, const char* clientId, } catch (...) { // Assuming only the functor call can throw exceptions, we need // to unlock the current key before exiting - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); throw; } } @@ -314,16 +314,16 @@ LockableMap<Map>::each(const Functor& functor, const char* clientId, mapped_type val; Decision decision; { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); if (findNextKey(key, val, clientId, guard) || key > last) return; _lockedKeys.insert(LockId(key, clientId)); } try{ while (true) { decision = functor(const_cast<const key_type&>(key), val); - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); if (handleDecision(key, val, decision)) return; ++key; if (findNextKey(key, val, clientId, guard) || key > last) return; @@ -332,9 +332,9 @@ LockableMap<Map>::each(const Functor& functor, const char* clientId, } catch (...) { // Assuming only the functor call can throw exceptions, we need // to unlock the current key before exiting - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, clientId)); - guard.broadcast(); + _cond.notify_all(); throw; } } @@ -347,7 +347,7 @@ LockableMap<Map>::all(Functor& functor, const char* clientId, { key_type key = first; mapped_type val; - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); while (true) { if (findNextKey(key, val, clientId, guard) || key > last) return; Decision d(functor(const_cast<const key_type&>(key), val)); @@ -364,7 +364,7 @@ LockableMap<Map>::all(const Functor& functor, const char* clientId, { key_type key = first; mapped_type val; - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); while (true) { if (findNextKey(key, val, clientId, guard) || key > last) return; Decision d(functor(const_cast<const key_type&>(key), val)); @@ -383,7 +383,7 @@ LockableMap<Map>::processNextChunk(Functor& functor, const uint32_t chunkSize) { mapped_type val; - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); for (uint32_t processed = 0; processed < chunkSize; ++processed) { if (findNextKey(key, val, clientId, guard)) { return false; @@ -422,7 +422,7 @@ void LockableMap<Map>::print(std::ostream& out, bool verbose, const std::string& indent) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); out << "LockableMap {\n" << indent << " "; if (verbose) { @@ -462,9 +462,9 @@ template<typename Map> void LockableMap<Map>::unlock(const key_type& key) { - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _lockedKeys.erase(LockId(key, "")); - guard.broadcast(); + _cond.notify_all(); } /** @@ -550,7 +550,7 @@ LockableMap<Map>::addAndLockResults( const std::vector<BucketId::Type> keys, const char* clientId, std::map<BucketId, WrappedEntry>& results, - vespalib::MonitorGuard& guard) + std::unique_lock<std::mutex> &guard) { // Wait until all buckets are free to be added, then add them all. while (true) { @@ -567,7 +567,7 @@ LockableMap<Map>::addAndLockResults( if (!allOk) { typename LockWaiters::Key waitId(_lockWaiters.insert(LockId(waitingFor, clientId))); - guard.wait(); + _cond.wait(guard); _lockWaiters.erase(waitId); } else { for (uint32_t i=0; i<keys.size(); i++) { @@ -593,7 +593,7 @@ LockableMap<Map>::createAppropriateBucket( const char* clientId, const BucketId& bucket) { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); typename Map::const_iterator iter = _map.lower_bound(bucket.toKey()); // Find the two buckets around the possible new bucket. The new @@ -613,7 +613,7 @@ LockableMap<Map>::createAppropriateBucket( BucketId::Type key = newBucket.stripUnused().toKey(); LockId lid(key, clientId); - ackquireKey(lid, guard); + acquireKey(lid, guard); bool preExisted; typename Map::iterator it = _map.find(key, true, preExisted); _lockedKeys.insert(LockId(key, clientId)); @@ -625,7 +625,7 @@ std::map<document::BucketId, typename LockableMap<Map>::WrappedEntry> LockableMap<Map>::getContained(const BucketId& bucket, const char* clientId) { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); std::map<BucketId, WrappedEntry> results; BucketId result; @@ -718,7 +718,7 @@ std::map<document::BucketId, typename LockableMap<Map>::WrappedEntry> LockableMap<Map>::getAll(const BucketId& bucket, const char* clientId, const BucketId& sibling) { - vespalib::MonitorGuard guard(_lock); + std::unique_lock<std::mutex> guard(_lock); std::map<BucketId, WrappedEntry> results; std::vector<BucketId::Type> keys; @@ -734,7 +734,7 @@ template<typename Map> bool LockableMap<Map>::isConsistent(const typename LockableMap<Map>::WrappedEntry& entry) { - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); BucketId sibling(0); std::vector<BucketId::Type> keys; @@ -750,7 +750,7 @@ template<typename Map> void LockableMap<Map>::showLockClients(vespalib::asciistream & out) const { - vespalib::MonitorGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); out << "Currently grabbed locks:"; for (typename LockIdSet::const_iterator it = _lockedKeys.begin(); it != _lockedKeys.end(); ++it) diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp index 3a832f0fe3b..fc2c2066b6f 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp @@ -5,6 +5,7 @@ #include "config-stor-bucket-init.h" #include "storbucketdb.h" #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/io/fileutil.h> @@ -65,9 +66,8 @@ StorageBucketDBInitializer::System::System( : _doneInitializeHandler(doneInitializeHandler), _component(compReg, "storagebucketdbinitializer"), _partitions(partitions), - _bucketDatabase(_component.getBucketDatabase(BucketSpace::placeHolder())), + _bucketSpaceRepo(_component.getBucketSpaceRepo()), _nodeIndex(_component.getIndex()), - _distribution(*_component.getDistribution()), _nodeState() { // Is this correct? We should get the node state from the node state updater @@ -82,6 +82,12 @@ StorageBucketDBInitializer::System::System( } } +StorBucketDatabase & +StorageBucketDBInitializer::System::getBucketDatabase(document::BucketSpace bucketSpace) const +{ + return _component.getBucketDatabase(bucketSpace); +} + StorageBucketDBInitializer::Metrics::Metrics(framework::Component& component) : metrics::MetricSet("dbinit", "", "Metrics for the storage bucket database initializer"), @@ -134,7 +140,10 @@ StorageBucketDBInitializer::StorageBucketDBInitializer( // Initialize read state for disks being available for (uint32_t i=0; i<_system._partitions.size(); ++i) { if (!_system._partitions[i].isUp()) continue; - _readState[i] = BucketReadState::UP(new BucketReadState); + _readState[i] = std::make_unique<BucketSpaceReadState>(); + for (const auto &elem : _system._bucketSpaceRepo) { + _readState[i]->emplace(elem.first, std::make_unique<BucketReadState>()); + } _state._dirsToList += 1; } _system._component.registerStatusPage(*this); @@ -155,9 +164,14 @@ StorageBucketDBInitializer::onOpen() // Trigger bucket database initialization for (uint32_t i=0; i<_system._partitions.size(); ++i) { if (!_system._partitions[i].isUp()) continue; - ReadBucketList::SP msg(new ReadBucketList(BucketSpace::placeHolder(), spi::PartitionId(i))); - _state._lists[msg->getMsgId()] = msg; - sendDown(msg); + assert(_readState[i]); + const BucketSpaceReadState &spaceState = *_readState[i]; + for (const auto &stateElem : spaceState) { + document::BucketSpace bucketSpace = stateElem.first; + auto msg = std::make_shared<ReadBucketList>(bucketSpace, spi::PartitionId(i)); + _state._lists[msg->getMsgId()] = msg; + sendDown(msg); + } } framework::MilliSecTime maxProcessingTime(10); framework::MilliSecTime sleepTime(1000); @@ -220,6 +234,26 @@ StorageBucketDBInitializer::print( out << "StorageBucketDBInitializer()"; } +namespace { + +size_t +notDoneCount(const StorageBucketDBInitializer::ReadState &readState) +{ + size_t result = 0; + for (const auto &elem : readState) { + if (elem) { + for (const auto &stateElem : *elem) { + if (!stateElem.second->_done) { + ++result; + } + } + } + } + return result; +} + +} + void StorageBucketDBInitializer::reportHtmlStatus( std::ostream& out, const framework::HttpUrlPath&) const @@ -261,10 +295,7 @@ StorageBucketDBInitializer::reportHtmlStatus( out << " " << _state._infoRequests.size() << " info requests pending.<br/>\n"; } - uint32_t incompleteScan = 0; - for (uint32_t i=0; i<_readState.size(); ++i) { - if (_readState[i].get() != 0 && !_readState[i]->_done) ++incompleteScan; - } + uint32_t incompleteScan = notDoneCount(_readState); if (incompleteScan == 0) { out << " Done iterating bucket database to generate info " << "requests.<br/>\n"; @@ -304,29 +335,31 @@ StorageBucketDBInitializer::reportHtmlStatus( out << " <h3>Disk " << i << " is down</h3>\n"; continue; } - BucketReadState& state(*_readState[i]); - out << " <h3>Disk " << i << "</h3>\n"; - out << " Pending info requests: " << pendingCounts[i] << " ("; - if (state._pending.empty()) { - out << "none"; - } else { - bool first = true; - for (BucketSet::const_iterator it = state._pending.begin(); - it != state._pending.end(); ++it) - { - if (!first) { - out << ", "; - } else { - first = false; + const BucketSpaceReadState& spaceState(*_readState[i]); + for (const auto &stateElem : spaceState) { + const BucketReadState &state = *stateElem.second; + out << " <h3>Disk " << i << ", bucket space " << stateElem.first.getId() << "</h3>\n"; + out << " Pending info requests: " << pendingCounts[i] << " ("; + if (state._pending.empty()) { + out << "none"; + } else { + bool first = true; + for (BucketSet::const_iterator it = state._pending.begin(); + it != state._pending.end(); ++it) { + if (!first) { + out << ", "; + } else { + first = false; + } + out << *it; } - out << *it; } + out << ")<br/>\n"; + out << " Bucket database iterator: " << state._databaseIterator + << "<br/>\n"; + out << " Done iterating bucket database. " + << (state._done ? "true" : "false") << "<br/>\n"; } - out << ")<br/>\n"; - out << " Bucked database iterator: " << state._databaseIterator - << "<br/>\n"; - out << " Done iterating bucket database. " - << (state._done ? "true" : "false") << "<br/>\n"; } for (std::map<Disk, uint32_t>::iterator it = pendingCounts.begin(); it != pendingCounts.end(); ++it) @@ -338,11 +371,12 @@ StorageBucketDBInitializer::reportHtmlStatus( // Always called from worker thread. Worker monitor already grabbed void StorageBucketDBInitializer::registerBucket(const document::Bucket &bucket, + const lib::Distribution &distribution, spi::PartitionId partition, api::BucketInfo bucketInfo) { document::BucketId bucketId(bucket.getBucketId()); - StorBucketDatabase::WrappedEntry entry(_system._bucketDatabase.get( + StorBucketDatabase::WrappedEntry entry(_system.getBucketDatabase(bucket.getBucketSpace()).get( bucketId, "StorageBucketDBInitializer::registerBucket", StorBucketDatabase::CREATE_IF_NONEXISTING)); if (bucketInfo.valid()) { @@ -369,7 +403,7 @@ StorageBucketDBInitializer::registerBucket(const document::Bucket &bucket, return; } uint32_t keepOnDisk, joinFromDisk; - if (_system._distribution.getPreferredAvailableDisk( + if (distribution.getPreferredAvailableDisk( _system._nodeState, _system._nodeIndex, bucketId.stripUnused()) == partition) { @@ -384,8 +418,7 @@ StorageBucketDBInitializer::registerBucket(const document::Bucket &bucket, bucketId.toString().c_str(), entry->disk, int(partition), keepOnDisk); entry.unlock(); // Must not have bucket db lock while sending down - InternalBucketJoinCommand::SP cmd(new InternalBucketJoinCommand( - bucket, keepOnDisk, joinFromDisk)); + auto cmd = std::make_shared<InternalBucketJoinCommand>(bucket, keepOnDisk, joinFromDisk); { _state._joins[cmd->getMsgId()] = cmd; } @@ -396,7 +429,7 @@ StorageBucketDBInitializer::registerBucket(const document::Bucket &bucket, bucketId.toString().c_str(), int(partition)); entry->disk = partition; entry.write(); - uint16_t disk(_system._distribution.getIdealDisk( + uint16_t disk(distribution.getIdealDisk( _system._nodeState, _system._nodeIndex, bucketId.stripUnused(), lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN)); if (disk != partition) { @@ -459,9 +492,11 @@ namespace { // Always called from worker thread. It holds worker monitor. void -StorageBucketDBInitializer::sendReadBucketInfo(spi::PartitionId disk) +StorageBucketDBInitializer::sendReadBucketInfo(spi::PartitionId disk, document::BucketSpace bucketSpace) { - BucketReadState& state(*_readState[disk]); + auto itr = _readState[disk]->find(bucketSpace); + assert(itr != _readState[disk]->end()); + BucketReadState& state = *itr->second; if (state._done || state._pending.size() >= _config._maxPendingInfoReadsPerDisk) { @@ -473,7 +508,7 @@ StorageBucketDBInitializer::sendReadBucketInfo(spi::PartitionId disk) NextBucketOnDiskFinder finder(disk, state._databaseIterator, count); LOG(spam, "Iterating bucket db further. Starting at iterator %s", state._databaseIterator.toString().c_str()); - _system._bucketDatabase.all(finder, + _system.getBucketDatabase(bucketSpace).all(finder, "StorageBucketDBInitializer::readBucketInfo", state._databaseIterator.stripUnused().toKey()); if (finder._alreadySet > 0) { @@ -481,8 +516,8 @@ StorageBucketDBInitializer::sendReadBucketInfo(spi::PartitionId disk) _state._infoSetByLoad += finder._alreadySet; } for (uint32_t i=0; i<finder._next.size(); ++i) { - document::Bucket bucket(BucketSpace::placeHolder(), finder._next[i]); - ReadBucketInfo::SP cmd(new ReadBucketInfo(bucket)); + document::Bucket bucket(bucketSpace, finder._next[i]); + auto cmd = std::make_shared<ReadBucketInfo>(bucket); cmd->setPriority(_config._infoReadPriority); state._pending.insert(finder._next[i]); _state._infoRequests[cmd->getMsgId()] = disk; @@ -586,14 +621,16 @@ StorageBucketDBInitializer::handleReadBucketListReply( const spi::BucketIdListResult::List& list(reply.getBuckets()); api::BucketInfo info; assert(!info.valid()); + const auto &contentBucketSpace(_system._bucketSpaceRepo.get(reply.getBucketSpace())); + auto distribution(contentBucketSpace.getDistribution()); for (uint32_t i=0, n=list.size(); i<n; ++i) { - registerBucket(document::Bucket(reply.getBucketSpace(), list[i]), reply.getPartition(), info); + registerBucket(document::Bucket(reply.getBucketSpace(), list[i]), *distribution, reply.getPartition(), info); } if (++_state._dirsListed == _state._dirsToList) { handleListingCompleted(); } checkIfDone(); - sendReadBucketInfo(reply.getPartition()); + sendReadBucketInfo(reply.getPartition(), reply.getBucketSpace()); } // Always called from worker thread. It holds worker monitor. @@ -601,12 +638,13 @@ void StorageBucketDBInitializer::handleReadBucketInfoReply( ReadBucketInfoReply& reply) { + document::BucketSpace bucketSpace = reply.getBucket().getBucketSpace(); if (reply.getResult().failed()) { LOGBP(warning, "Deleting %s from bucket database. Cannot use it as we " "failed to read bucket info for it: %s", reply.getBucketId().toString().c_str(), reply.getResult().toString().c_str()); - _system._bucketDatabase.erase(reply.getBucketId(), + _system.getBucketDatabase(bucketSpace).erase(reply.getBucketId(), "dbinit.failedreply"); } _metrics._infoReadCount.inc(); @@ -622,7 +660,9 @@ StorageBucketDBInitializer::handleReadBucketInfoReply( } else { uint32_t disk(it->second); _state._infoRequests.erase(it->first); - BucketReadState& state(*_readState[disk]); + auto itr = _readState[disk]->find(bucketSpace); + assert(itr != _readState[disk]->end()); + BucketReadState& state = *itr->second; BucketSet::iterator it2(state._pending.find(reply.getBucketId())); if (it2 == state._pending.end()) { LOGBP(warning, "Got bucket info reply for %s that was registered " @@ -632,12 +672,12 @@ StorageBucketDBInitializer::handleReadBucketInfoReply( state._pending.erase(reply.getBucketId()); LOG(spam, "Got info reply for %s: %s", reply.getBucketId().toString().c_str(), - _system._bucketDatabase.get( + _system.getBucketDatabase(reply.getBucket().getBucketSpace()).get( reply.getBucketId(), "dbinit.inforeply") ->getBucketInfo().toString().c_str()); } checkIfDone(); - sendReadBucketInfo(spi::PartitionId(disk)); + sendReadBucketInfo(spi::PartitionId(disk), bucketSpace); } } @@ -661,7 +701,7 @@ StorageBucketDBInitializer::handleInternalBucketJoinReply( LOG(debug, "Completed internal bucket join for %s. Got bucket info %s", reply.getBucketId().toString().c_str(), reply.getBucketInfo().toString().c_str()); - StorBucketDatabase::WrappedEntry entry(_system._bucketDatabase.get( + StorBucketDatabase::WrappedEntry entry(_system.getBucketDatabase(reply.getBucket().getBucketSpace()).get( reply.getBucketId(), "StorageBucketDBInitializer::onInternalBucketJoinReply")); entry->setBucketInfo(reply.getBucketInfo()); @@ -674,6 +714,16 @@ StorageBucketDBInitializer::handleInternalBucketJoinReply( checkIfDone(); } +namespace { + +bool +isDone(const StorageBucketDBInitializer::ReadState &readState) +{ + return notDoneCount(readState) == 0; +} + +} + // Always called from worker thread. It holds worker monitor. void StorageBucketDBInitializer::checkIfDone() @@ -681,8 +731,8 @@ StorageBucketDBInitializer::checkIfDone() if (_state._dirsListed < _state._dirsToList) return; if (!_state._infoRequests.empty()) return; if (!_state._joins.empty()) return; - for (uint32_t i=0; i<_readState.size(); ++i) { - if (_readState[i].get() != 0 && !_readState[i]->_done) return; + if (!isDone(_readState)) { + return; } _state._doneInitializing = true; _system._doneInitializeHandler.notifyDoneInitializing(); @@ -698,17 +748,19 @@ StorageBucketDBInitializer::calculateMinProgressFromDiskIterators() const if (_readState[disk].get() == 0) { continue; } - const BucketReadState& state(*_readState[disk]); - document::BucketId bid(state._databaseIterator); + for (const auto &stateElem : *_readState[disk]) { + const BucketReadState &state = *stateElem.second; + document::BucketId bid(state._databaseIterator); - double progress; - if (!state._done) { - progress = BucketProgressCalculator::calculateProgress(bid); - } else { - progress = 1.0; - } + double progress; + if (!state._done) { + progress = BucketProgressCalculator::calculateProgress(bid); + } else { + progress = 1.0; + } - minProgress = std::min(minProgress, progress); + minProgress = std::min(minProgress, progress); + } } //std::cerr << "minProgress: " << minProgress << "\n"; return minProgress; diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h index 99f273a384a..57b95e14f48 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h @@ -51,6 +51,7 @@ #include <vespa/vdslib/state/nodestate.h> #include <vespa/config/subscription/configuri.h> #include <list> +#include <unordered_map> namespace storage { @@ -77,9 +78,8 @@ class StorageBucketDBInitializer : public StorageLink, DoneInitializeHandler& _doneInitializeHandler; ServiceLayerComponent _component; const spi::PartitionStateList& _partitions; - StorBucketDatabase& _bucketDatabase; + const ContentBucketSpaceRepo& _bucketSpaceRepo; uint32_t _nodeIndex; - lib::Distribution& _distribution; lib::NodeState _nodeState; // Disk info for ideal state calculations framework::Thread::UP _thread; @@ -87,6 +87,8 @@ class StorageBucketDBInitializer : public StorageLink, DoneInitializeHandler& doneInitializeHandler, ServiceLayerComponentRegister&, const Config&); + + StorBucketDatabase &getBucketDatabase(document::BucketSpace bucketSpace) const; }; struct Metrics : public metrics::MetricSet { metrics::LongCountMetric _wrongDisk; @@ -126,12 +128,17 @@ class StorageBucketDBInitializer : public StorageLink, ~GlobalState(); }; +public: + using BucketSpaceReadState = std::unordered_map<document::BucketSpace, + std::unique_ptr<BucketReadState>, document::BucketSpace::hash>; + using ReadState = std::vector<std::unique_ptr<BucketSpaceReadState>>; +private: Config _config; System _system; Metrics _metrics; GlobalState _state; - std::vector<std::unique_ptr<BucketReadState>> _readState; + ReadState _readState; public: StorageBucketDBInitializer(const config::ConfigUri&, @@ -180,13 +187,14 @@ public: std::vector<uint32_t>& path); /** Register a bucket in the bucket database. */ void registerBucket(const document::Bucket &bucket, + const lib::Distribution &distribution, spi::PartitionId, api::BucketInfo bucketInfo); /** * Sends more read bucket info to a given disk. Lock must already be taken. * Will be released by function prior to sending messages down. */ - void sendReadBucketInfo(spi::PartitionId); + void sendReadBucketInfo(spi::PartitionId, document::BucketSpace bucketSpace); /** Check whether initialization is complete. Should hold lock to call it.*/ void checkIfDone(); diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp index b9073ca2cdc..bc9a8b0c428 100644 --- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp +++ b/storage/src/vespa/storage/bucketmover/bucketmover.cpp @@ -4,6 +4,7 @@ #include "htmltable.h" #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/common/bucketmessages.h> +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/storageutil/log.h> #include <vespa/config/common/exceptions.h> @@ -27,7 +28,7 @@ BucketMover::BucketMover(const config::ConfigUri & configUri, _cycleCount(0), _nextRun(0), _configFetcher(configUri.getContext()), - _diskDistribution(_component.getDistribution()->getDiskDistribution()), + _diskDistribution(currentDiskDistribution()), _maxSleepTime(60 * 60) { if (!configUri.empty()) { @@ -40,7 +41,7 @@ BucketMover::BucketMover(const config::ConfigUri & configUri, BucketMover::~BucketMover() { - if (_thread.get() != 0) { + if (_thread) { LOG(error, "BucketMover deleted without calling close() first"); onClose(); } @@ -61,10 +62,10 @@ BucketMover::onClose() // Avoid getting config during shutdown _configFetcher.close(); // Close thread to ensure we don't send anything more down after - if (_thread.get()) { + if (_thread) { _thread->interruptAndJoin(&_wait); LOG(debug, "Bucket mover worker thread closed."); - _thread.reset(0); + _thread.reset(); } } @@ -111,12 +112,14 @@ BucketMover::startNewRun() // If not in a run but time to start another one, do so LOG(debug, "Starting new move cycle at time %s.", _component.getClock().getTimeInSeconds().toString().c_str()); - _currentRun.reset(new bucketmover::Run( - _component.getBucketDatabase(BucketSpace::placeHolder()), - _component.getDistribution(), + // TODO consider if we should invoke bucket moving across all bucket spaces. Not likely to ever be needed. + // If so, we have to spawn off an individual Run per space, as it encompasses + // both a (disk) distribution and a bucket database. + _currentRun = std::make_unique<bucketmover::Run>( + _component.getBucketSpaceRepo().get(document::BucketSpace::placeHolder()), *_component.getStateUpdater().getReportedNodeState(), _component.getIndex(), - _component.getClock())); + _component.getClock()); } void @@ -124,8 +127,7 @@ BucketMover::queueNewMoves() { // If we have too few pending, send some new moves, if there are more // moves to perform. - while (_pendingMoves.size() < uint32_t(_config->maxPending)) - { + while (_pendingMoves.size() < uint32_t(_config->maxPending)) { Move nextMove = _currentRun->getNextMove(); // If no more moves to do, stop attempting to send more. @@ -133,11 +135,8 @@ BucketMover::queueNewMoves() break; } _pendingMoves.push_back(nextMove); - document::Bucket bucket(BucketSpace::placeHolder(), nextMove.getBucketId()); - std::shared_ptr<BucketDiskMoveCommand> cmd( - new BucketDiskMoveCommand(bucket, - nextMove.getSourceDisk(), - nextMove.getTargetDisk())); + auto cmd = std::make_shared<BucketDiskMoveCommand>( + nextMove.getBucket(), nextMove.getSourceDisk(), nextMove.getTargetDisk()); cmd->setPriority(nextMove.getPriority()); _newMoves.push_back(cmd); } @@ -171,11 +170,9 @@ BucketMover::finishCurrentRun() void BucketMover::sendNewMoves() { - for (std::list<BucketDiskMoveCommand::SP>::iterator it - = _newMoves.begin(); it != _newMoves.end(); ++it) - { - LOG(debug, "Moving bucket: %s", (**it).toString().c_str()); - sendDown(*it); + for (auto& move : _newMoves) { + LOG(debug, "Moving bucket: %s", move->toString().c_str()); + sendDown(move); // Be able to sleep a bit between moves for debugging to see // what is happening. (Cannot use wait() here as reply of @@ -196,7 +193,7 @@ BucketMover::tick() framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); - if (_currentRun.get() == 0) { + if (!_currentRun) { if (currentTime >= _nextRun) { startNewRun(); } else { @@ -287,25 +284,24 @@ bool BucketMover::onInternalReply( const std::shared_ptr<api::InternalReply>& internalReply) { - // We only care about move disk bucket replies - std::shared_ptr<BucketDiskMoveReply> reply( - std::dynamic_pointer_cast<BucketDiskMoveReply>(internalReply)); - if (!reply.get()) return false; + // We only care about move disk bucket replies + auto reply = std::dynamic_pointer_cast<BucketDiskMoveReply>(internalReply); + if (!reply) { + return false; + } - // Warn if we see move replies outside of a run. Should not be possible. + // Warn if we see move replies outside of a run. Should not be possible. vespalib::MonitorGuard monitor(_wait); - if (_currentRun.get() == 0) { + if (!_currentRun) { LOG(warning, "Got a bucket disk move reply while no run is active. " "This should not happen, as runs should stay active until " "all requests are answered."); return true; } - // Match move against pending ones + // Match move against pending ones Move move; - for (std::list<Move>::iterator it = _pendingMoves.begin(); - it != _pendingMoves.end(); ++it) - { - if (it->getBucketId() == reply->getBucketId() + for (auto it = _pendingMoves.begin(); it != _pendingMoves.end(); ++it) { + if (it->getBucket() == reply->getBucket() && it->getSourceDisk() == reply->getSrcDisk() && it->getTargetDisk() == reply->getDstDisk()) { @@ -338,18 +334,20 @@ BucketMover::onInternalReply( return true; } +// TODO if we start supporting disk moves for other spaces than the default space +// we also have to check all disk distributions here. void BucketMover::storageDistributionChanged() { - lib::Distribution::SP distribution = _component.getDistribution(); - - // Verify that the actual disk distribution changed, if not ignore - lib::Distribution::DiskDistribution newDistr(distribution->getDiskDistribution()); + // Verify that the actual disk distribution changed, if not ignore + lib::Distribution::DiskDistribution newDistr(currentDiskDistribution()); - if (_diskDistribution == newDistr) return; + if (_diskDistribution == newDistr) { + return; + } vespalib::MonitorGuard monitor(_wait); - if (_currentRun.get() != 0) { + if (_currentRun) { LOG(info, "Aborting bucket mover run as disk distribution changed " "from %s to %s.", lib::Distribution::getDiskDistributionName(_diskDistribution).c_str(), @@ -365,9 +363,14 @@ BucketMover::storageDistributionChanged() _nextRun = framework::SecondTime(0); } +lib::Distribution::DiskDistribution BucketMover::currentDiskDistribution() const { + auto distribution = _component.getBucketSpaceRepo().get(document::BucketSpace::placeHolder()).getDistribution(); + return distribution->getDiskDistribution(); +} + bool BucketMover::isWorkingOnCycle() const { vespalib::MonitorGuard monitor(_wait); - return (_currentRun.get() != 0); + return (_currentRun.get() != nullptr); } uint32_t BucketMover::getCycleCount() const { @@ -382,7 +385,7 @@ BucketMover::print(std::ostream& out, bool verbose, (void) verbose; (void) indent; vespalib::MonitorGuard monitor(_wait); out << "BucketMover() {"; - if (_currentRun.get() != 0) { + if (_currentRun) { out << "\n" << indent << " "; _currentRun->print(out, verbose, indent + " "); } else { @@ -390,11 +393,9 @@ BucketMover::print(std::ostream& out, bool verbose, } if (verbose && !_history.empty()) { out << "\n" << indent << " History:"; - for (std::list<RunStatistics>::const_iterator it = _history.begin(); - it != _history.end(); ++it) - { + for (auto& entry : _history) { out << "\n" << indent << " "; - it->print(out, true, indent + " "); + entry.print(out, true, indent + " "); } } out << "\n" << indent << "}"; @@ -412,17 +413,14 @@ BucketMover::reportHtmlStatus(std::ostream& out, printCurrentStatus(out, *_history.begin()); } out << "<h2>Current move cycle</h2>\n"; - if (_currentRun.get() != 0) { + if (_currentRun) { printRunHtml(out, *_currentRun); if (_currentRun->getPendingMoves().empty()) { out << "<blockquote>No pending moves.</blockquote>\n"; } else { out << "<blockquote>Pending bucket moves:<ul>\n"; - for (std::list<Move>::const_iterator it - = _currentRun->getPendingMoves().begin(); - it != _currentRun->getPendingMoves().end(); ++it) - { - out << "<li>" << *it << "</li>\n"; + for (auto& entry : _currentRun->getPendingMoves()) { + out << "<li>" << entry << "</li>\n"; } out << "</ul></blockquote>\n"; } @@ -432,7 +430,7 @@ BucketMover::reportHtmlStatus(std::ostream& out, framework::SecondTime currentTime( _component.getClock().getTimeInSeconds()); if (_nextRun <= currentTime) { - if (_thread.get() != 0) { + if (_thread) { out << "Next run to start immediately."; // Wake up thread, so user sees it starts immediately :) monitor.signal(); @@ -454,10 +452,8 @@ BucketMover::reportHtmlStatus(std::ostream& out, } if (!_history.empty()) { out << "<h2>Statistics from previous bucket mover cycles</h2>\n"; - for (std::list<RunStatistics>::const_iterator it = _history.begin(); - it != _history.end(); ++it) - { - printRunStatisticsHtml(out, *it); + for (auto& entry : _history) { + printRunStatisticsHtml(out, entry); } } } diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.h b/storage/src/vespa/storage/bucketmover/bucketmover.h index 0b9cfc04455..42cbf693fcb 100644 --- a/storage/src/vespa/storage/bucketmover/bucketmover.h +++ b/storage/src/vespa/storage/bucketmover/bucketmover.h @@ -83,6 +83,7 @@ private: void run(framework::ThreadHandle&) override; bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; void storageDistributionChanged() override; + lib::Distribution::DiskDistribution currentDiskDistribution() const; framework::SecondTime calculateWaitTimeOfNextRun() const; diff --git a/storage/src/vespa/storage/bucketmover/move.cpp b/storage/src/vespa/storage/bucketmover/move.cpp index 7d6f96c46ba..b91329fad36 100644 --- a/storage/src/vespa/storage/bucketmover/move.cpp +++ b/storage/src/vespa/storage/bucketmover/move.cpp @@ -9,13 +9,13 @@ namespace bucketmover { Move::Move() : _sourceDisk(0), _targetDisk(0), - _bucket(0), + _bucket(document::BucketSpace::placeHolder(), document::BucketId(0)), _totalDocSize(0), _priority(255) { } -Move::Move(uint16_t source, uint16_t target, const document::BucketId& bucket, +Move::Move(uint16_t source, uint16_t target, const document::Bucket& bucket, uint32_t totalDocSize) : _sourceDisk(source), _targetDisk(target), diff --git a/storage/src/vespa/storage/bucketmover/move.h b/storage/src/vespa/storage/bucketmover/move.h index e2401e5abed..92d05e4e0ae 100644 --- a/storage/src/vespa/storage/bucketmover/move.h +++ b/storage/src/vespa/storage/bucketmover/move.h @@ -8,7 +8,7 @@ #pragma once -#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucket.h> #include <vespa/vespalib/util/printable.h> namespace storage { @@ -17,22 +17,22 @@ namespace bucketmover { class Move : public vespalib::Printable { uint16_t _sourceDisk; uint16_t _targetDisk; - document::BucketId _bucket; + document::Bucket _bucket; uint32_t _totalDocSize; uint8_t _priority; public: Move(); - Move(uint16_t source, uint16_t target, const document::BucketId& bucket, + Move(uint16_t source, uint16_t target, const document::Bucket& bucket, uint32_t totalDocSize); /** False if invalid move. (Empty constructor) Indicates end of run. */ - bool isDefined() const { return (_bucket.getRawId() != 0); } + bool isDefined() const { return (_bucket.getBucketId().getRawId() != 0); } // Only valid to call if move is defined uint16_t getSourceDisk() const { return _sourceDisk; } uint16_t getTargetDisk() const { return _targetDisk; } - const document::BucketId& getBucketId() const { return _bucket; } + const document::Bucket& getBucket() const { return _bucket; } uint8_t getPriority() const { return _priority; } uint32_t getTotalDocSize() const { return _totalDocSize; } diff --git a/storage/src/vespa/storage/bucketmover/run.cpp b/storage/src/vespa/storage/bucketmover/run.cpp index 6a0ef2079ce..22bcfa55f15 100644 --- a/storage/src/vespa/storage/bucketmover/run.cpp +++ b/storage/src/vespa/storage/bucketmover/run.cpp @@ -12,24 +12,24 @@ LOG_SETUP(".bucketmover.run"); namespace storage { namespace bucketmover { -Run::Run(StorBucketDatabase& db, - lib::Distribution::SP distribution, +Run::Run(ContentBucketSpace& bucketSpace, const lib::NodeState& nodeState, uint16_t nodeIndex, framework::Clock& clock) - : _bucketDatabase(db), - _distribution(distribution), + : _bucketSpace(bucketSpace), + _distribution(bucketSpace.getDistribution()), _nodeState(nodeState), _nodeIndex(nodeIndex), _entries(), _iterationDone(false), - _statistics(distribution->getDiskDistribution(), clock, nodeState), + _statistics(_distribution->getDiskDistribution(), clock, nodeState), _aborted(false) { } namespace { struct BucketIterator { + document::BucketSpace _iteratedBucketSpace; const lib::Distribution& _distribution; const lib::NodeState& _nodeState; RunStatistics& _statistics; @@ -39,10 +39,12 @@ namespace { uint32_t _bucketsVisited; document::BucketId _firstBucket; - BucketIterator(const lib::Distribution& d, const lib::NodeState& ns, + BucketIterator(document::BucketSpace iteratedBucketSpace, + const lib::Distribution& d, const lib::NodeState& ns, uint16_t nodeIndex, RunStatistics& stats, std::list<Move>& entries) - : _distribution(d), + : _iteratedBucketSpace(iteratedBucketSpace), + _distribution(d), _nodeState(ns), _statistics(stats), _entries(entries), @@ -57,12 +59,12 @@ namespace { operator()(document::BucketId::Type revId, StorBucketDatabase::Entry& entry) { - document::BucketId bucket(document::BucketId::keyToBucketId(revId)); - if (bucket == _firstBucket) { + document::BucketId bucketId(document::BucketId::keyToBucketId(revId)); + if (bucketId == _firstBucket) { return StorBucketDatabase::CONTINUE; } uint16_t idealDisk = _distribution.getIdealDisk( - _nodeState, _nodeIndex, bucket, + _nodeState, _nodeIndex, bucketId, lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN); RunStatistics::DiskData& diskData( _statistics._diskData[entry.disk]); @@ -72,10 +74,11 @@ namespace { diskData._bucketSize += entry.getBucketInfo().getTotalDocumentSize(); ++diskData._bucketsFoundOnCorrectDisk; } else { + document::Bucket bucket(_iteratedBucketSpace, bucketId); _entries.push_back(Move( entry.disk, idealDisk, bucket, entry.getBucketInfo().getTotalDocumentSize())); } - _statistics._lastBucketVisited = bucket; + _statistics._lastBucketVisited = bucketId; if (++_bucketsVisited >= _maxBucketsToIterateAtOnce) { return StorBucketDatabase::ABORT; } @@ -104,18 +107,16 @@ Run::getNextMove() if (!_statistics._diskData[e.getTargetDisk()]._diskDisabled) { _pending.push_back(e); - _statistics._lastBucketProcessed = e.getBucketId(); - _statistics._lastBucketProcessedTime - = _statistics._clock->getTimeInSeconds(); + _statistics._lastBucketProcessed = e.getBucket(); // Only used for printing + _statistics._lastBucketProcessedTime = _statistics._clock->getTimeInSeconds(); return e; } } // Cache more entries - BucketIterator it(*_distribution, _nodeState, _nodeIndex, _statistics, - _entries); - _bucketDatabase.all(it, "bucketmover::Run", - _statistics._lastBucketVisited.toKey()); + BucketIterator it(_bucketSpace.bucketSpace(), *_distribution, + _nodeState, _nodeIndex, _statistics, _entries); + _bucketSpace.bucketDatabase().all(it, "bucketmover::Run", _statistics._lastBucketVisited.toKey()); if (it._bucketsVisited == 0) { _iterationDone = true; if (_pending.empty()) { @@ -128,31 +129,6 @@ Run::getNextMove() } void -Run::depleteMoves() -{ - while (true) { - // Cache more entries - BucketIterator bi(*_distribution, _nodeState, _nodeIndex, _statistics, - _entries); - _bucketDatabase.all(bi, "bucketmover::depleteMoves", - _statistics._lastBucketVisited.toKey()); - if (bi._bucketsVisited == 0) { - break; - } - for (std::list<Move>::const_iterator it = _entries.begin(); - it != _entries.end(); ++it) - { - ++_statistics._diskData[it->getSourceDisk()][it->getTargetDisk()] - ._bucketsLeftOnWrongDisk; - uint32_t size = it->getTotalDocSize(); - _statistics._diskData[it->getSourceDisk()]._bucketSize += size; - } - _entries.clear(); - } - finalize(); -} - -void Run::finalize() { _statistics._endTime = _statistics._clock->getTimeInSeconds(); @@ -162,10 +138,8 @@ void Run::removePending(Move& move) { bool foundPending = false; - for (std::list<Move>::iterator it = _pending.begin(); it != _pending.end(); - ++it) - { - if (it->getBucketId() == move.getBucketId()) { + for (auto it = _pending.begin(); it != _pending.end(); ++it) { + if (it->getBucket() == move.getBucket()) { _pending.erase(it); foundPending = true; break; @@ -173,7 +147,7 @@ Run::removePending(Move& move) } if (!foundPending) { LOG(warning, "Got answer for %s that was not in the pending list.", - move.getBucketId().toString().c_str()); + move.getBucket().toString().c_str()); return; } if (_iterationDone && _pending.empty()) { diff --git a/storage/src/vespa/storage/bucketmover/run.h b/storage/src/vespa/storage/bucketmover/run.h index 11f2cf0763c..eb7a6df2d17 100644 --- a/storage/src/vespa/storage/bucketmover/run.h +++ b/storage/src/vespa/storage/bucketmover/run.h @@ -18,6 +18,7 @@ #include "move.h" #include "runstatistics.h" +#include <vespa/storage/common/content_bucket_space.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/nodestate.h> #include <list> @@ -33,8 +34,8 @@ class Clock; namespace bucketmover { class Run : public document::Printable { - StorBucketDatabase& _bucketDatabase; - lib::Distribution::SP _distribution; + ContentBucketSpace& _bucketSpace; + std::shared_ptr<const lib::Distribution> _distribution; lib::NodeState _nodeState; uint16_t _nodeIndex; uint32_t _maxEntriesToKeep; @@ -48,8 +49,7 @@ class Run : public document::Printable { public: Run(const Run &) = delete; Run & operator = (const Run &) = delete; - Run(StorBucketDatabase&, - lib::Distribution::SP, + Run(ContentBucketSpace& bucketSpace, const lib::NodeState&, uint16_t nodeIndex, framework::Clock&); @@ -78,12 +78,6 @@ public: */ Move getNextMove(); - /** - * Run through the database not doing any moves. Useful to do a run only - * to gather statistics of current state. - */ - void depleteMoves(); - void moveOk(Move& move); void moveFailedBucketNotFound(Move& move); void moveFailed(Move& move); diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.cpp b/storage/src/vespa/storage/bucketmover/runstatistics.cpp index 8f7fe67fcf3..314f04a0d66 100644 --- a/storage/src/vespa/storage/bucketmover/runstatistics.cpp +++ b/storage/src/vespa/storage/bucketmover/runstatistics.cpp @@ -39,7 +39,7 @@ RunStatistics::RunStatistics(DiskDistribution d, framework::Clock& clock, const lib::NodeState& ns) : _clock(&clock), _distribution(d), - _lastBucketProcessed(0), + _lastBucketProcessed(), _lastBucketVisited(0), _diskData(ns.getDiskCount(), DiskData(ns.getDiskCount())), _startTime(_clock->getTimeInSeconds()), @@ -149,13 +149,14 @@ RunStatistics::getWronglyPlacedRatio() const return static_cast<double>(wrong) / total; } +// FIXME does not cover multiple spaces (but only used for printing) double RunStatistics::getProgress() const { if (_endTime.isSet()) return 1.0; double result = 0; double weight = 0.5; - uint64_t key = _lastBucketProcessed.toKey(); + uint64_t key = _lastBucketProcessed.getBucketId().toKey(); for (uint16_t i=0; i<64; ++i) { uint64_t flag = uint64_t(1) << (63 - i); if ((key & flag) == flag) { diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.h b/storage/src/vespa/storage/bucketmover/runstatistics.h index da51be9ef7d..908f345b307 100644 --- a/storage/src/vespa/storage/bucketmover/runstatistics.h +++ b/storage/src/vespa/storage/bucketmover/runstatistics.h @@ -38,7 +38,7 @@ #include <vespa/vdslib/state/nodestate.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucket.h> #include <vespa/vespalib/util/printable.h> #include <vespa/storageframework/generic/clock/time.h> @@ -76,7 +76,7 @@ struct RunStatistics : public document::Printable { framework::Clock* _clock; DiskDistribution _distribution; - document::BucketId _lastBucketProcessed; + document::Bucket _lastBucketProcessed; document::BucketId _lastBucketVisited; // Invalid bucket for starting point std::vector<DiskData> _diskData; framework::SecondTime _startTime; diff --git a/storage/src/vespa/storage/common/bucket_resolver.h b/storage/src/vespa/storage/common/bucket_resolver.h new file mode 100644 index 00000000000..f1e334807bf --- /dev/null +++ b/storage/src/vespa/storage/common/bucket_resolver.h @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucket.h> +#include <vespa/vespalib/stllike/string.h> + +namespace document { class DocumentId; } + +namespace storage { + +/** + * Interface for resolving which bucket a given a document id belongs to. + */ +struct BucketResolver { + virtual ~BucketResolver() {} + virtual document::Bucket bucketFromId(const document::DocumentId &documentId) const = 0; + virtual document::BucketSpace bucketSpaceFromName(const vespalib::string &bucketSpace) const = 0; + virtual vespalib::string nameFromBucketSpace(const document::BucketSpace &bucketSpace) const = 0; +}; + +} diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index 1d9d64ad24f..3157bad49e5 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -39,6 +39,12 @@ ReadBucketListReply::ReadBucketListReply(const ReadBucketList& cmd) ReadBucketListReply::~ReadBucketListReply() { } +document::Bucket +ReadBucketListReply::getBucket() const +{ + return document::Bucket(_bucketSpace, document::BucketId()); +} + void ReadBucketListReply::print(std::ostream& out, bool verbose, const std::string& indent) const { diff --git a/storage/src/vespa/storage/common/bucketmessages.h b/storage/src/vespa/storage/common/bucketmessages.h index 0ff7a22aa4d..941928b1064 100644 --- a/storage/src/vespa/storage/common/bucketmessages.h +++ b/storage/src/vespa/storage/common/bucketmessages.h @@ -55,6 +55,7 @@ public: document::BucketSpace getBucketSpace() const { return _bucketSpace; } spi::PartitionId getPartition() const { return _partition; } + document::Bucket getBucket() const override; spi::BucketIdListResult::List& getBuckets() { return _buckets; } const spi::BucketIdListResult::List& getBuckets() const { diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp index b78be81c9de..4344bccc785 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space.cpp @@ -4,9 +4,26 @@ namespace storage { -ContentBucketSpace::ContentBucketSpace() - : _bucketDatabase() +ContentBucketSpace::ContentBucketSpace(document::BucketSpace bucketSpace) + : _bucketSpace(bucketSpace), + _bucketDatabase(), + _lock(), + _distribution() { } +void +ContentBucketSpace::setDistribution(std::shared_ptr<const lib::Distribution> distribution) +{ + std::lock_guard<std::mutex> guard(_lock); + _distribution = std::move(distribution); +} + +std::shared_ptr<const lib::Distribution> +ContentBucketSpace::getDistribution() const +{ + std::lock_guard<std::mutex> guard(_lock); + return _distribution; +} + } diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h index 2efb2eca06d..3b3dddade4f 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.h +++ b/storage/src/vespa/storage/common/content_bucket_space.h @@ -1,21 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/document/bucket/bucketspace.h> #include <vespa/storage/bucketdb/storbucketdb.h> +#include <mutex> namespace storage { +namespace lib { class Distribution; } + /** * Class representing a bucket space (with associated bucket database) on a content node. */ class ContentBucketSpace { private: + document::BucketSpace _bucketSpace; StorBucketDatabase _bucketDatabase; + mutable std::mutex _lock; + std::shared_ptr<const lib::Distribution> _distribution; public: using UP = std::unique_ptr<ContentBucketSpace>; - ContentBucketSpace(); + ContentBucketSpace(document::BucketSpace bucketSpace); + + document::BucketSpace bucketSpace() const noexcept { return _bucketSpace; } StorBucketDatabase &bucketDatabase() { return _bucketDatabase; } + void setDistribution(std::shared_ptr<const lib::Distribution> distribution); + std::shared_ptr<const lib::Distribution> getDistribution() const; }; } diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp index 04e2c4c27d3..1846c132c0a 100644 --- a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp @@ -9,7 +9,7 @@ namespace storage { ContentBucketSpaceRepo::ContentBucketSpaceRepo() : _map() { - _map.emplace(BucketSpace::placeHolder(), std::make_unique<ContentBucketSpace>()); + _map.emplace(BucketSpace::placeHolder(), std::make_unique<ContentBucketSpace>(BucketSpace::placeHolder())); } ContentBucketSpace & @@ -21,6 +21,16 @@ ContentBucketSpaceRepo::get(BucketSpace bucketSpace) const return *itr->second; } +ContentBucketSpaceRepo::BucketSpaces +ContentBucketSpaceRepo::getBucketSpaces() const +{ + BucketSpaces result; + for (const auto &elem : _map) { + result.push_back(elem.first); + } + return result; +} + size_t ContentBucketSpaceRepo::getBucketMemoryUsage() const { diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.h b/storage/src/vespa/storage/common/content_bucket_space_repo.h index 390cfc15f5d..0d4ddb86bcf 100644 --- a/storage/src/vespa/storage/common/content_bucket_space_repo.h +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.h @@ -13,6 +13,7 @@ namespace storage { class ContentBucketSpaceRepo { public: using BucketSpaceMap = std::unordered_map<document::BucketSpace, ContentBucketSpace::UP, document::BucketSpace::hash>; + using BucketSpaces = std::vector<document::BucketSpace>; private: BucketSpaceMap _map; @@ -23,6 +24,7 @@ public: BucketSpaceMap::const_iterator begin() const { return _map.begin(); } BucketSpaceMap::const_iterator end() const { return _map.end(); } + BucketSpaces getBucketSpaces() const; size_t getBucketMemoryUsage() const; template <typename Functor> diff --git a/storage/src/vespa/storage/common/servicelayercomponent.cpp b/storage/src/vespa/storage/common/servicelayercomponent.cpp index 68c41536f97..11311a4d189 100644 --- a/storage/src/vespa/storage/common/servicelayercomponent.cpp +++ b/storage/src/vespa/storage/common/servicelayercomponent.cpp @@ -28,7 +28,7 @@ ServiceLayerComponent::getBucketDatabase(BucketSpace bucketSpace) const uint16_t ServiceLayerComponent::getIdealPartition(const document::Bucket& bucket) const { - return getDistribution()->getIdealDisk( + return _bucketSpaceRepo->get(bucket.getBucketSpace()).getDistribution()->getIdealDisk( *getStateUpdater().getReportedNodeState(), getIndex(), bucket.getBucketId(), lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN); } @@ -37,7 +37,7 @@ uint16_t ServiceLayerComponent::getPreferredAvailablePartition( const document::Bucket& bucket) const { - return getDistribution()->getPreferredAvailableDisk( + return _bucketSpaceRepo->get(bucket.getBucketSpace()).getDistribution()->getPreferredAvailableDisk( *getStateUpdater().getReportedNodeState(), getIndex(), bucket.getBucketId()); } diff --git a/storage/src/vespa/storage/config/CMakeLists.txt b/storage/src/vespa/storage/config/CMakeLists.txt index 4a20d510043..65eeeaf3221 100644 --- a/storage/src/vespa/storage/config/CMakeLists.txt +++ b/storage/src/vespa/storage/config/CMakeLists.txt @@ -28,3 +28,5 @@ vespa_generate_config(storage_storageconfig stor-prioritymapping.def) install_config_definition(stor-prioritymapping.def vespa.config.content.core.stor-prioritymapping.def) vespa_generate_config(storage_storageconfig rpc-provider.def) install_config_definition(rpc-provider.def vespa.config.content.core.rpc-provider.def) +vespa_generate_config(storage_storageconfig bucketspaces.def) +install_config_definition(bucketspaces.def vespa.config.content.core.bucketspaces.def) diff --git a/storage/src/vespa/storage/config/bucketspaces.def b/storage/src/vespa/storage/config/bucketspaces.def new file mode 100644 index 00000000000..3ed1abba0b4 --- /dev/null +++ b/storage/src/vespa/storage/config/bucketspaces.def @@ -0,0 +1,11 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## This config contains the document types handled by a given content cluster +## and the bucket space they belong to. + +## The name of a document type. +documenttype[].name string + +## The bucket space this document type belongs to. +documenttype[].bucketspace string diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 3dc08d858ad..8adbfcaf9da 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_library(storage_distributor bucketlistmerger.cpp clusterinformation.cpp distributor_bucket_space.cpp - distributor_bucket_space_component.cpp distributor_bucket_space_repo.cpp distributor.cpp distributor_host_info_reporter.cpp @@ -27,6 +26,7 @@ vespa_add_library(storage_distributor operationtargetresolverimpl.cpp ownership_transfer_safe_time_point_calculator.cpp pendingclusterstate.cpp + pending_bucket_space_db_transition.cpp pendingmessagetracker.cpp persistence_operation_metric_set.cpp persistencemessagetracker.cpp diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 569136b8b10..46fa0f72d76 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -2,6 +2,8 @@ #include "bucketdbupdater.h" #include "distributor.h" +#include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include "simpleclusterinformation.h" #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/persistence.h> @@ -20,13 +22,12 @@ namespace storage::distributor { BucketDBUpdater::BucketDBUpdater(Distributor& owner, DistributorBucketSpaceRepo &bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, DistributorMessageSender& sender, DistributorComponentRegister& compReg) : framework::StatusReporter("bucketdb", "Bucket DB Updater"), - _bucketSpaceComponent(owner, bucketSpaceRepo, bucketSpace, compReg, "Bucket DB Updater"), + _distributorComponent(owner, bucketSpaceRepo, compReg, "Bucket DB Updater"), _sender(sender), - _transitionTimer(_bucketSpaceComponent.getClock()) + _transitionTimer(_distributorComponent.getClock()) { } @@ -59,13 +60,11 @@ BucketDBUpdater::hasPendingClusterState() const } BucketOwnership -BucketDBUpdater::checkOwnershipInPendingState(const document::BucketId& b) const +BucketDBUpdater::checkOwnershipInPendingState(const document::Bucket& b) const { if (hasPendingClusterState()) { const lib::ClusterState& state(_pendingClusterState->getNewClusterState()); - const lib::Distribution& distribution(_pendingClusterState->getDistribution()); - document::Bucket bucket(BucketSpace::placeHolder(), b); - if (!_bucketSpaceComponent.ownsBucketInState(distribution, state, bucket)) { + if (!_distributorComponent.ownsBucketInState(state, b)) { return BucketOwnership::createNotOwnedInState(state); } } @@ -75,18 +74,18 @@ BucketDBUpdater::checkOwnershipInPendingState(const document::BucketId& b) const void BucketDBUpdater::sendRequestBucketInfo( uint16_t node, - const document::BucketId& bucket, + const document::Bucket& bucket, const std::shared_ptr<MergeReplyGuard>& mergeReplyGuard) { - if (!_bucketSpaceComponent.storageNodeIsUp(node)) { + if (!_distributorComponent.storageNodeIsUp(node)) { return; } std::vector<document::BucketId> buckets; - buckets.push_back(bucket); + buckets.push_back(bucket.getBucketId()); std::shared_ptr<api::RequestBucketInfoCommand> msg( - new api::RequestBucketInfoCommand(BucketSpace::placeHolder(), buckets)); + new api::RequestBucketInfoCommand(bucket.getBucketSpace(), buckets)); LOG(debug, "Sending request bucket info command %lu for " @@ -96,40 +95,43 @@ BucketDBUpdater::sendRequestBucketInfo( node); msg->setPriority(50); - msg->setAddress(_bucketSpaceComponent.nodeAddress(node)); + msg->setAddress(_distributorComponent.nodeAddress(node)); _sentMessages[msg->getMsgId()] = - BucketRequest(node, _bucketSpaceComponent.getUniqueTimestamp(), + BucketRequest(node, _distributorComponent.getUniqueTimestamp(), bucket, mergeReplyGuard); _sender.sendCommand(msg); } void BucketDBUpdater::recheckBucketInfo(uint32_t nodeIdx, - const document::BucketId& bid) + const document::Bucket& bucket) { - sendRequestBucketInfo(nodeIdx, bid, std::shared_ptr<MergeReplyGuard>()); + sendRequestBucketInfo(nodeIdx, bucket, std::shared_ptr<MergeReplyGuard>()); } void BucketDBUpdater::removeSuperfluousBuckets( - const lib::Distribution& newDistribution, const lib::ClusterState& newState) { - // Remove all buckets not belonging to this distributor, or - // being on storage nodes that are no longer up. - NodeRemover proc( - _bucketSpaceComponent.getClusterState(), - newState, - _bucketSpaceComponent.getBucketIdFactory(), - _bucketSpaceComponent.getIndex(), - newDistribution, - _bucketSpaceComponent.getDistributor().getStorageNodeUpStates()); - - _bucketSpaceComponent.getBucketDatabase().forEach(proc); - - for (const auto & entry :proc.getBucketsToRemove()) { - _bucketSpaceComponent.getBucketDatabase().remove(entry); + for (auto &elem : _distributorComponent.getBucketSpaceRepo()) { + const auto &newDistribution(elem.second->getDistribution()); + auto &bucketDb(elem.second->getBucketDatabase()); + + // Remove all buckets not belonging to this distributor, or + // being on storage nodes that are no longer up. + NodeRemover proc( + _distributorComponent.getClusterState(), + newState, + _distributorComponent.getBucketIdFactory(), + _distributorComponent.getIndex(), + newDistribution, + _distributorComponent.getDistributor().getStorageNodeUpStates()); + bucketDb.forEach(proc); + + for (const auto & entry :proc.getBucketsToRemove()) { + bucketDb.remove(entry); + } } } @@ -140,37 +142,35 @@ BucketDBUpdater::ensureTransitionTimerStarted() // that will make transition times appear artificially low. if (!hasPendingClusterState()) { _transitionTimer = framework::MilliSecTimer( - _bucketSpaceComponent.getClock()); + _distributorComponent.getClock()); } } void BucketDBUpdater::completeTransitionTimer() { - _bucketSpaceComponent.getDistributor().getMetrics() + _distributorComponent.getDistributor().getMetrics() .stateTransitionTime.addValue(_transitionTimer.getElapsedTimeAsDouble()); } void -BucketDBUpdater::storageDistributionChanged( - const lib::Distribution& distribution) +BucketDBUpdater::storageDistributionChanged() { ensureTransitionTimerStarted(); - removeSuperfluousBuckets(distribution, - _bucketSpaceComponent.getClusterState()); + removeSuperfluousBuckets(_distributorComponent.getClusterState()); ClusterInformation::CSP clusterInfo(new SimpleClusterInformation( - _bucketSpaceComponent.getIndex(), - distribution, - _bucketSpaceComponent.getClusterState(), - _bucketSpaceComponent.getDistributor().getStorageNodeUpStates())); + _distributorComponent.getIndex(), + _distributorComponent.getClusterState(), + _distributorComponent.getDistributor().getStorageNodeUpStates())); _pendingClusterState = PendingClusterState::createForDistributionChange( - _bucketSpaceComponent.getClock(), + _distributorComponent.getClock(), std::move(clusterInfo), _sender, - _bucketSpaceComponent.getUniqueTimestamp()); - _outdatedNodes = _pendingClusterState->getOutdatedNodeSet(); + _distributorComponent.getBucketSpaceRepo(), + _distributorComponent.getUniqueTimestamp()); + _outdatedNodesMap = _pendingClusterState->getOutdatedNodesMap(); } void @@ -179,7 +179,7 @@ BucketDBUpdater::replyToPreviousPendingClusterStateIfAny() if (_pendingClusterState.get() && _pendingClusterState->getCommand().get()) { - _bucketSpaceComponent.sendUp( + _distributorComponent.sendUp( std::make_shared<api::SetSystemStateReply>(*_pendingClusterState->getCommand())); } } @@ -192,7 +192,7 @@ BucketDBUpdater::onSetSystemState( "Received new cluster state %s", cmd->getSystemState().toString().c_str()); - lib::ClusterState oldState = _bucketSpaceComponent.getClusterState(); + lib::ClusterState oldState = _distributorComponent.getClusterState(); const lib::ClusterState& state = cmd->getSystemState(); if (state == oldState) { @@ -200,26 +200,24 @@ BucketDBUpdater::onSetSystemState( } ensureTransitionTimerStarted(); - removeSuperfluousBuckets( - _bucketSpaceComponent.getDistribution(), - cmd->getSystemState()); + removeSuperfluousBuckets(cmd->getSystemState()); replyToPreviousPendingClusterStateIfAny(); ClusterInformation::CSP clusterInfo( new SimpleClusterInformation( - _bucketSpaceComponent.getIndex(), - _bucketSpaceComponent.getDistribution(), - _bucketSpaceComponent.getClusterState(), - _bucketSpaceComponent.getDistributor() + _distributorComponent.getIndex(), + _distributorComponent.getClusterState(), + _distributorComponent.getDistributor() .getStorageNodeUpStates())); _pendingClusterState = PendingClusterState::createForClusterStateChange( - _bucketSpaceComponent.getClock(), + _distributorComponent.getClock(), std::move(clusterInfo), _sender, + _distributorComponent.getBucketSpaceRepo(), cmd, - _outdatedNodes, - _bucketSpaceComponent.getUniqueTimestamp()); - _outdatedNodes = _pendingClusterState->getOutdatedNodeSet(); + _outdatedNodesMap, + _distributorComponent.getUniqueTimestamp()); + _outdatedNodesMap = _pendingClusterState->getOutdatedNodesMap(); if (isPendingClusterStateCompleted()) { processCompletedPendingClusterState(); @@ -246,7 +244,7 @@ BucketDBUpdater::onMergeBucketReply( // bucket again to make sure it's ok. for (uint32_t i = 0; i < reply->getNodes().size(); i++) { sendRequestBucketInfo(reply->getNodes()[i].index, - reply->getBucketId(), + reply->getBucket(), replyGuard); } @@ -256,7 +254,7 @@ BucketDBUpdater::onMergeBucketReply( void BucketDBUpdater::enqueueRecheckUntilPendingStateEnabled( uint16_t node, - const document::BucketId& bucket) + const document::Bucket& bucket) { LOG(spam, "DB updater has a pending cluster state, enqueuing recheck " @@ -303,10 +301,10 @@ BucketDBUpdater::onNotifyBucketChange( if (hasPendingClusterState()) { enqueueRecheckUntilPendingStateEnabled(cmd->getSourceIndex(), - cmd->getBucketId()); + cmd->getBucket()); } else { sendRequestBucketInfo(cmd->getSourceIndex(), - cmd->getBucketId(), + cmd->getBucket(), std::shared_ptr<MergeReplyGuard>()); } @@ -355,8 +353,8 @@ BucketDBUpdater::handleSingleBucketInfoFailure( LOG(debug, "Request bucket info failed towards node %d: error was %s", req.targetNode, repl->getResult().toString().c_str()); - if (req.bucket != document::BucketId(0)) { - framework::MilliSecTime sendTime(_bucketSpaceComponent.getClock()); + if (req.bucket.getBucketId() != document::BucketId(0)) { + framework::MilliSecTime sendTime(_distributorComponent.getClock()); sendTime += framework::MilliSecTime(100); _delayedRequests.emplace_back(sendTime, req); } @@ -369,7 +367,7 @@ BucketDBUpdater::resendDelayedMessages() _pendingClusterState->resendDelayedMessages(); } if (_delayedRequests.empty()) return; // Don't fetch time if not needed - framework::MilliSecTime currentTime(_bucketSpaceComponent.getClock()); + framework::MilliSecTime currentTime(_distributorComponent.getClock()); while (!_delayedRequests.empty() && currentTime >= _delayedRequests.front().first) { @@ -407,7 +405,7 @@ BucketDBUpdater::mergeBucketInfoWithDatabase( std::sort(newList.begin(), newList.end(), sort_pred); BucketListMerger merger(newList, existing, req.timestamp); - updateDatabase(req.targetNode, merger); + updateDatabase(req.bucket.getBucketSpace(), req.targetNode, merger); } bool @@ -424,7 +422,7 @@ BucketDBUpdater::processSingleBucketInfoReply( BucketRequest req = iter->second; _sentMessages.erase(iter); - if (!_bucketSpaceComponent.storageNodeIsUp(req.targetNode)) { + if (!_distributorComponent.storageNodeIsUp(req.targetNode)) { // Ignore replies from nodes that are down. return true; } @@ -449,11 +447,12 @@ BucketDBUpdater::addBucketInfoForNode( } void -BucketDBUpdater::findRelatedBucketsInDatabase(uint16_t node, const document::BucketId& bucketId, +BucketDBUpdater::findRelatedBucketsInDatabase(uint16_t node, const document::Bucket& bucket, BucketListMerger::BucketList& existing) { + auto &distributorBucketSpace(_distributorComponent.getBucketSpaceRepo().get(bucket.getBucketSpace())); std::vector<BucketDatabase::Entry> entries; - _bucketSpaceComponent.getBucketDatabase().getAll(bucketId, entries); + distributorBucketSpace.getBucketDatabase().getAll(bucket.getBucketId(), entries); for (const BucketDatabase::Entry & entry : entries) { addBucketInfoForNode(entry, node, existing); @@ -461,16 +460,16 @@ BucketDBUpdater::findRelatedBucketsInDatabase(uint16_t node, const document::Buc } void -BucketDBUpdater::updateDatabase(uint16_t node, BucketListMerger& merger) +BucketDBUpdater::updateDatabase(document::BucketSpace bucketSpace, uint16_t node, BucketListMerger& merger) { for (const document::BucketId & bucketId : merger.getRemovedEntries()) { - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); - _bucketSpaceComponent.removeNodeFromDB(bucket, node); + document::Bucket bucket(bucketSpace, bucketId); + _distributorComponent.removeNodeFromDB(bucket, node); } for (const BucketListMerger::BucketEntry& entry : merger.getAddedEntries()) { - document::Bucket bucket(BucketSpace::placeHolder(), entry.first); - _bucketSpaceComponent.updateBucketDatabase( + document::Bucket bucket(bucketSpace, entry.first); + _distributorComponent.updateBucketDatabase( bucket, BucketCopy(merger.getTimestamp(), node, entry.second), DatabaseUpdate::CREATE_IF_NONEXISTING); @@ -486,19 +485,19 @@ BucketDBUpdater::isPendingClusterStateCompleted() const void BucketDBUpdater::processCompletedPendingClusterState() { - _pendingClusterState->mergeInto(_bucketSpaceComponent.getBucketDatabase()); + _pendingClusterState->mergeIntoBucketDatabases(); if (_pendingClusterState->getCommand().get()) { enableCurrentClusterStateInDistributor(); - _bucketSpaceComponent.getDistributor().getMessageSender().sendDown( + _distributorComponent.getDistributor().getMessageSender().sendDown( _pendingClusterState->getCommand()); addCurrentStateToClusterStateHistory(); } else { - _bucketSpaceComponent.getDistributor().notifyDistributionChangeEnabled(); + _distributorComponent.getDistributor().notifyDistributionChangeEnabled(); } _pendingClusterState.reset(); - _outdatedNodes.clear(); + _outdatedNodesMap.clear(); sendAllQueuedBucketRechecks(); completeTransitionTimer(); } @@ -513,7 +512,7 @@ BucketDBUpdater::enableCurrentClusterStateInDistributor() "BucketDBUpdater finished processing state %s", state.toString().c_str()); - _bucketSpaceComponent.getDistributor().enableClusterState(state); + _distributorComponent.getDistributor().enableClusterState(state); } void @@ -564,7 +563,7 @@ BucketDBUpdater::reportXmlStatus(vespalib::xml::XmlOutputStream& xos, using namespace vespalib::xml; xos << XmlTag("bucketdb") << XmlTag("systemstate_active") - << XmlContent(_bucketSpaceComponent.getClusterState().toString()) + << XmlContent(_distributorComponent.getClusterState().toString()) << XmlEndTag(); if (_pendingClusterState) { xos << *_pendingClusterState; @@ -583,10 +582,10 @@ BucketDBUpdater::reportXmlStatus(vespalib::xml::XmlOutputStream& xos, { xos << XmlTag("storagenode") << XmlAttribute("index", entry.second.targetNode); - if (entry.second.bucket.getRawId() == 0) { + if (entry.second.bucket.getBucketId().getRawId() == 0) { xos << XmlAttribute("bucket", ALL); } else { - xos << XmlAttribute("bucket", entry.second.bucket.getId(), XmlAttribute::HEX); + xos << XmlAttribute("bucket", entry.second.bucket.getBucketId().getId(), XmlAttribute::HEX); } xos << XmlAttribute("sendtimestamp", entry.second.timestamp) << XmlEndTag(); diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index 994e207f200..a3c9804c2b4 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -6,8 +6,8 @@ #include "distributorcomponent.h" #include "distributormessagesender.h" #include "pendingclusterstate.h" -#include "distributor_bucket_space_component.h" -#include <vespa/document/bucket/bucketid.h> +#include "outdated_nodes_map.h" +#include <vespa/document/bucket/bucket.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vdslib/state/clusterstate.h> @@ -27,29 +27,30 @@ class BucketDBUpdater : public framework::StatusReporter, public api::MessageHandler { public: + using OutdatedNodes = dbtransition::OutdatedNodes; + using OutdatedNodesMap = dbtransition::OutdatedNodesMap; BucketDBUpdater(Distributor& owner, DistributorBucketSpaceRepo &bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, DistributorMessageSender& sender, DistributorComponentRegister& compReg); ~BucketDBUpdater(); void flush(); - BucketOwnership checkOwnershipInPendingState(const document::BucketId&) const; - void recheckBucketInfo(uint32_t nodeIdx, const document::BucketId& bid); + BucketOwnership checkOwnershipInPendingState(const document::Bucket&) const; + void recheckBucketInfo(uint32_t nodeIdx, const document::Bucket& bucket); bool onSetSystemState(const std::shared_ptr<api::SetSystemStateCommand>& cmd) override; bool onRequestBucketInfoReply(const std::shared_ptr<api::RequestBucketInfoReply> & repl) override; bool onMergeBucketReply(const std::shared_ptr<api::MergeBucketReply>& reply) override; bool onNotifyBucketChange(const std::shared_ptr<api::NotifyBucketChangeCommand>&) override; void resendDelayedMessages(); - void storageDistributionChanged(const lib::Distribution&); + void storageDistributionChanged(); vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream&, const framework::HttpUrlPath&) const; vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const; - DistributorComponent& getDistributorComponent() { return _bucketSpaceComponent; } + DistributorComponent& getDistributorComponent() { return _distributorComponent; } /** * Returns whether the current PendingClusterState indicates that there has @@ -63,7 +64,7 @@ public: } private: - DistributorBucketSpaceComponent _bucketSpaceComponent; + DistributorComponent _distributorComponent; class MergeReplyGuard { public: MergeReplyGuard(BucketDBUpdater& updater, const std::shared_ptr<api::MergeBucketReply>& reply) @@ -81,9 +82,9 @@ private: struct BucketRequest { BucketRequest() - : targetNode(0), bucket(0), timestamp(0) {}; + : targetNode(0), bucket(), timestamp(0) {}; - BucketRequest(uint16_t t, uint64_t currentTime, const document::BucketId& b, + BucketRequest(uint16_t t, uint64_t currentTime, const document::Bucket& b, const std::shared_ptr<MergeReplyGuard>& guard) : targetNode(t), bucket(b), @@ -91,7 +92,7 @@ private: _mergeReplyGuard(guard) {}; uint16_t targetNode; - document::BucketId bucket; + document::Bucket bucket; uint64_t timestamp; std::shared_ptr<MergeReplyGuard> _mergeReplyGuard; @@ -99,11 +100,11 @@ private: struct EnqueuedBucketRecheck { uint16_t node; - document::BucketId bucket; + document::Bucket bucket; EnqueuedBucketRecheck() : node(0), bucket() {} - EnqueuedBucketRecheck(uint16_t _node, const document::BucketId& _bucket) + EnqueuedBucketRecheck(uint16_t _node, const document::Bucket& _bucket) : node(_node), bucket(_bucket) {} @@ -121,7 +122,6 @@ private: bool hasPendingClusterState() const; bool pendingClusterStateAccepted(const std::shared_ptr<api::RequestBucketInfoReply>& repl); - bool bucketOwnedAccordingToPendingState(const document::BucketId& bucketId) const; bool processSingleBucketInfoReply(const std::shared_ptr<api::RequestBucketInfoReply>& repl); void handleSingleBucketInfoFailure(const std::shared_ptr<api::RequestBucketInfoReply>& repl, const BucketRequest& req); @@ -131,7 +131,7 @@ private: const BucketRequest& req); void convertBucketInfoToBucketList(const std::shared_ptr<api::RequestBucketInfoReply>& repl, uint16_t targetNode, BucketListMerger::BucketList& newList); - void sendRequestBucketInfo(uint16_t node, const document::BucketId& bucket, + void sendRequestBucketInfo(uint16_t node, const document::Bucket& bucket, const std::shared_ptr<MergeReplyGuard>& mergeReply); void addBucketInfoForNode(const BucketDatabase::Entry& e, uint16_t node, BucketListMerger::BucketList& existing) const; @@ -143,24 +143,24 @@ private: * in bucketId, or that bucketId is contained in, that have copies * on the given node. */ - void findRelatedBucketsInDatabase(uint16_t node, const document::BucketId& bucketId, + void findRelatedBucketsInDatabase(uint16_t node, const document::Bucket& bucket, BucketListMerger::BucketList& existing); /** Updates the bucket database from the information generated by the given bucket list merger. */ - void updateDatabase(uint16_t node, BucketListMerger& merger); + void updateDatabase(document::BucketSpace bucketSpace, uint16_t node, BucketListMerger& merger); void updateState(const lib::ClusterState& oldState, const lib::ClusterState& newState); - void removeSuperfluousBuckets(const lib::Distribution& newDistribution, const lib::ClusterState& newState); + void removeSuperfluousBuckets(const lib::ClusterState& newState); void replyToPreviousPendingClusterStateIfAny(); void enableCurrentClusterStateInDistributor(); void addCurrentStateToClusterStateHistory(); - void enqueueRecheckUntilPendingStateEnabled(uint16_t node, const document::BucketId&); + void enqueueRecheckUntilPendingStateEnabled(uint16_t node, const document::Bucket&); void sendAllQueuedBucketRechecks(); friend class BucketDBUpdater_Test; @@ -226,7 +226,7 @@ private: std::list<PendingClusterState::Summary> _history; DistributorMessageSender& _sender; std::set<EnqueuedBucketRecheck> _enqueuedRechecks; - std::unordered_set<uint16_t> _outdatedNodes; + OutdatedNodesMap _outdatedNodesMap; framework::MilliSecTimer _transitionTimer; }; diff --git a/storage/src/vespa/storage/distributor/clusterinformation.cpp b/storage/src/vespa/storage/distributor/clusterinformation.cpp index 8e956a1cf61..cd09e4f46d4 100644 --- a/storage/src/vespa/storage/distributor/clusterinformation.cpp +++ b/storage/src/vespa/storage/distributor/clusterinformation.cpp @@ -6,45 +6,6 @@ namespace storage::distributor { -bool -ClusterInformation::ownsBucket(const document::BucketId& bucketId) const -{ - try { - uint16_t distributor(getDistribution().getIdealDistributorNode( - getClusterState(), bucketId)); - - return (getDistributorIndex() == distributor); - } catch (lib::TooFewBucketBitsInUseException& e) { - return false; - } catch (lib::NoDistributorsAvailableException& e) { - return false; - } -} - -bool -ClusterInformation::nodeInSameGroupAsSelf(uint16_t otherNode) const -{ - return (getDistribution().getNodeGraph().getGroupForNode(otherNode) - == getDistribution().getNodeGraph().getGroupForNode(getDistributorIndex())); -} - -vespalib::string -ClusterInformation::getDistributionHash() const -{ - return getDistribution().getNodeGraph().getDistributionConfigHash(); -} - -std::vector<uint16_t> -ClusterInformation::getIdealStorageNodesForState( - const lib::ClusterState& clusterState, - const document::BucketId& bucketId) const -{ - return getDistribution().getIdealStorageNodes( - clusterState, - bucketId, - getStorageUpStates()); -} - uint16_t ClusterInformation::getStorageNodeCount() const { diff --git a/storage/src/vespa/storage/distributor/clusterinformation.h b/storage/src/vespa/storage/distributor/clusterinformation.h index 4494b137f89..25f303d0f52 100644 --- a/storage/src/vespa/storage/distributor/clusterinformation.h +++ b/storage/src/vespa/storage/distributor/clusterinformation.h @@ -26,22 +26,10 @@ public: virtual uint16_t getDistributorIndex() const = 0; - virtual const lib::Distribution& getDistribution() const = 0; - virtual const lib::ClusterState& getClusterState() const = 0; virtual const char* getStorageUpStates() const = 0; - bool ownsBucket(const document::BucketId& bucketId) const; - - bool nodeInSameGroupAsSelf(uint16_t otherNode) const; - - vespalib::string getDistributionHash() const; - - std::vector<uint16_t> getIdealStorageNodesForState( - const lib::ClusterState& clusterState, - const document::BucketId& bucketId) const; - uint16_t getStorageNodeCount() const; }; diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 2de29697733..1edcbe75dd6 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -6,6 +6,7 @@ #include "idealstatemetricsset.h" #include "ownership_transfer_safe_time_point_calculator.h" #include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> @@ -72,12 +73,12 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _operationOwner(*this, _component.getClock()), _maintenanceOperationOwner(*this, _component.getClock()), _pendingMessageTracker(compReg), - _bucketDBUpdater(*this, *_bucketSpaceRepo, getDefaultBucketSpace(), *this, compReg), + _bucketDBUpdater(*this, *_bucketSpaceRepo, *this, compReg), _distributorStatusDelegate(compReg, *this, *this), _bucketDBStatusDelegate(compReg, *this, _bucketDBUpdater), - _idealStateManager(*this, *_bucketSpaceRepo, getDefaultBucketSpace(), compReg, + _idealStateManager(*this, *_bucketSpaceRepo, compReg, manageActiveBucketCopies), - _externalOperationHandler(*this, *_bucketSpaceRepo, getDefaultBucketSpace(), + _externalOperationHandler(*this, *_bucketSpaceRepo, _idealStateManager, compReg), _threadPool(threadPool), _initializingIsUp(true), @@ -87,7 +88,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _bucketPriorityDb(new SimpleBucketPriorityDatabase()), _scanner(new SimpleMaintenanceScanner( *_bucketPriorityDb, _idealStateManager, - getDefaultBucketSpace().getBucketDatabase())), + *_bucketSpaceRepo)), _throttlingStarter(new ThrottlingOperationStarter( _maintenanceOperationOwner)), _blockingStarter(new BlockingOperationStarter(_pendingMessageTracker, @@ -154,7 +155,7 @@ const DistributorBucketSpace& Distributor::getDefaultBucketSpace() const noexcep BucketOwnership Distributor::checkOwnershipInPendingState(const document::Bucket &b) const { - return _bucketDBUpdater.checkOwnershipInPendingState(b.getBucketId()); + return _bucketDBUpdater.checkOwnershipInPendingState(b); } void @@ -455,7 +456,7 @@ Distributor::storageDistributionChanged() void Distributor::recheckBucketInfo(uint16_t nodeIdx, const document::Bucket &bucket) { - _bucketDBUpdater.recheckBucketInfo(nodeIdx, bucket.getBucketId()); + _bucketDBUpdater.recheckBucketInfo(nodeIdx, bucket); } namespace { @@ -527,25 +528,13 @@ Distributor::checkBucketForSplit(document::BucketSpace bucketSpace, } Operation::SP operation = - _idealStateManager.generateInterceptingSplit(e, priority); + _idealStateManager.generateInterceptingSplit(bucketSpace, e, priority); if (operation.get()) { _maintenanceOperationOwner.start(operation, priority); } } -const lib::Distribution& -Distributor::getDistribution() const -{ - // FIXME having _distribution be mutable for this is smelly. Is this only - // in place for the sake of tests? - if (!_distribution.get()) { - _distribution = _component.getDistribution(); - } - - return *_distribution; -} - void Distributor::enableNextDistribution() { @@ -553,13 +542,13 @@ Distributor::enableNextDistribution() _distribution = _nextDistribution; propagateDefaultDistribution(_distribution); _nextDistribution = std::shared_ptr<lib::Distribution>(); - _bucketDBUpdater.storageDistributionChanged(getDistribution()); + _bucketDBUpdater.storageDistributionChanged(); } } void Distributor::propagateDefaultDistribution( - std::shared_ptr<lib::Distribution> distribution) + std::shared_ptr<const lib::Distribution> distribution) { _bucketSpaceRepo->setDefaultDistribution(std::move(distribution)); } @@ -684,9 +673,10 @@ Distributor::scanNextBucket() updateInternalMetricsForCompletedScan(); _scanner->reset(); } else { + const auto &distribution(_bucketSpaceRepo->get(scanResult.getBucketSpace()).getDistribution()); _bucketDBMetricUpdater.visit( scanResult.getEntry(), - _component.getDistribution()->getRedundancy()); + distribution.getRedundancy()); } return scanResult; } diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index f59b47574ba..9406dacf358 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -6,7 +6,6 @@ #include "bucketdbupdater.h" #include "pendingmessagetracker.h" #include "externaloperationhandler.h" -#include "maintenancebucket.h" #include "min_replica_provider.h" #include "distributorinterface.h" @@ -114,8 +113,6 @@ public: */ void checkBucketForSplit(document::BucketSpace bucketSpace, const BucketDatabase::Entry& e, uint8_t priority) override; - const lib::Distribution& getDistribution() const override; - const lib::ClusterState& getClusterState() const override { return _clusterState; } @@ -161,6 +158,8 @@ public: DistributorBucketSpace& getDefaultBucketSpace() noexcept; const DistributorBucketSpace& getDefaultBucketSpace() const noexcept; + DistributorBucketSpaceRepo &getBucketSpaceRepo() noexcept { return *_bucketSpaceRepo; } + const DistributorBucketSpaceRepo &getBucketSpaceRepo() const noexcept { return *_bucketSpaceRepo; } private: friend class Distributor_Test; @@ -232,7 +231,7 @@ private: Operation::SP& operation); void enableNextDistribution(); - void propagateDefaultDistribution(std::shared_ptr<lib::Distribution>); + void propagateDefaultDistribution(std::shared_ptr<const lib::Distribution>); lib::ClusterState _clusterState; @@ -251,7 +250,7 @@ private: IdealStateManager _idealStateManager; ExternalOperationHandler _externalOperationHandler; - mutable std::shared_ptr<lib::Distribution> _distribution; + std::shared_ptr<lib::Distribution> _distribution; std::shared_ptr<lib::Distribution> _nextDistribution; using MessageQueue = std::vector<std::shared_ptr<api::StorageMessage>>; diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index b33ff72a654..68fe9f441d7 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -5,7 +5,10 @@ namespace storage { namespace distributor { -DistributorBucketSpace::DistributorBucketSpace() { +DistributorBucketSpace::DistributorBucketSpace() + : _bucketDatabase(), + _distribution() +{ } DistributorBucketSpace::~DistributorBucketSpace() { diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index 17be92126cb..30893e8cfb1 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -26,7 +26,7 @@ namespace distributor { */ class DistributorBucketSpace { MapBucketDatabase _bucketDatabase; - std::shared_ptr<lib::Distribution> _distribution; + std::shared_ptr<const lib::Distribution> _distribution; public: DistributorBucketSpace(); ~DistributorBucketSpace(); @@ -43,7 +43,7 @@ public: return _bucketDatabase; } - void setDistribution(lib::Distribution::SP distribution) { + void setDistribution(std::shared_ptr<const lib::Distribution> distribution) { _distribution = std::move(distribution); } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_component.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_component.cpp deleted file mode 100644 index 4616179ae82..00000000000 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_component.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "distributor_bucket_space_component.h" - -namespace storage::distributor { - -DistributorBucketSpaceComponent::DistributorBucketSpaceComponent( - DistributorInterface& distributor, - DistributorBucketSpaceRepo &bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, - DistributorComponentRegister& compReg, - const std::string& name) - : DistributorComponent(distributor, bucketSpaceRepo, compReg, name), - _bucketSpace(bucketSpace) -{ -} - -} diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_component.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_component.h deleted file mode 100644 index 9c04cb6b67f..00000000000 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_component.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "distributorcomponent.h" -#include "distributor_bucket_space.h" - -namespace storage { -namespace distributor { - -/** - * Component bound to a specific bucket space, with utility operations to - * operate on buckets in this space. - */ -class DistributorBucketSpaceComponent : public DistributorComponent { - DistributorBucketSpace& _bucketSpace; -public: - DistributorBucketSpaceComponent(DistributorInterface& distributor, - DistributorBucketSpaceRepo &bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, - DistributorComponentRegister& compReg, - const std::string& name); - - BucketDatabase& getBucketDatabase() override { - return _bucketSpace.getBucketDatabase(); - } - - const BucketDatabase& getBucketDatabase() const override { - return _bucketSpace.getBucketDatabase(); - } - - const lib::Distribution& getDistribution() const override { - return _bucketSpace.getDistribution(); - } - -}; - -} -} diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index 67ca2397b11..d414f520bc2 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -1,43 +1,67 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/vdslib/distribution/distribution.h> #include <cassert> #include <vespa/log/log.h> -LOG_SETUP(".distributor.managed_bucket_space_repo"); +LOG_SETUP(".distributor.distributor_bucket_space_repo"); using document::BucketSpace; namespace storage { namespace distributor { -DistributorBucketSpaceRepo::DistributorBucketSpaceRepo() { +DistributorBucketSpaceRepo::DistributorBucketSpaceRepo() + : _map() +{ + add(BucketSpace::placeHolder(), std::make_unique<DistributorBucketSpace>()); } DistributorBucketSpaceRepo::~DistributorBucketSpaceRepo() { } +void +DistributorBucketSpaceRepo::add(document::BucketSpace bucketSpace, std::unique_ptr<DistributorBucketSpace> distributorBucketSpace) +{ + _map.emplace(bucketSpace, std::move(distributorBucketSpace)); +} + void DistributorBucketSpaceRepo::setDefaultDistribution( - std::shared_ptr<lib::Distribution> distr) + std::shared_ptr<const lib::Distribution> distr) { LOG(debug, "Got new default distribution '%s'", distr->toString().c_str()); // TODO all spaces, per-space config transforms - _defaultSpace.setDistribution(std::move(distr)); + getDefaultSpace().setDistribution(std::move(distr)); } DistributorBucketSpace & DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) { - assert(bucketSpace == BucketSpace::placeHolder()); - return _defaultSpace; + auto itr = _map.find(bucketSpace); + assert(itr != _map.end()); + return *itr->second; } const DistributorBucketSpace & DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) const { - assert(bucketSpace == BucketSpace::placeHolder()); - return _defaultSpace; + auto itr = _map.find(bucketSpace); + assert(itr != _map.end()); + return *itr->second; +} + +DistributorBucketSpace & +DistributorBucketSpaceRepo::getDefaultSpace() noexcept +{ + return get(BucketSpace::placeHolder()); +} + +const DistributorBucketSpace & +DistributorBucketSpaceRepo::getDefaultSpace() const noexcept +{ + return get(BucketSpace::placeHolder()); } } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h index 41eebf4bc4b..c3661b53e69 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h @@ -1,17 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "distributor_bucket_space.h" #include <vespa/document/bucket/bucketspace.h> #include <memory> +#include <unordered_map> namespace storage { +namespace lib { class Distribution; } + namespace distributor { +class DistributorBucketSpace; + class DistributorBucketSpaceRepo { - // TODO: multiple spaces. This is just to start re-wiring things. - DistributorBucketSpace _defaultSpace; +public: + using BucketSpaceMap = std::unordered_map<document::BucketSpace, std::unique_ptr<DistributorBucketSpace>, document::BucketSpace::hash>; + +private: + BucketSpaceMap _map; + public: DistributorBucketSpaceRepo(); ~DistributorBucketSpaceRepo(); @@ -21,14 +29,15 @@ public: DistributorBucketSpaceRepo(DistributorBucketSpaceRepo&&) = delete; DistributorBucketSpaceRepo& operator=(DistributorBucketSpaceRepo&&) = delete; - DistributorBucketSpace& getDefaultSpace() noexcept { return _defaultSpace; } - const DistributorBucketSpace& getDefaultSpace() const noexcept { - return _defaultSpace; - } + DistributorBucketSpace& getDefaultSpace() noexcept; + const DistributorBucketSpace& getDefaultSpace() const noexcept; DistributorBucketSpace &get(document::BucketSpace bucketSpace); const DistributorBucketSpace &get(document::BucketSpace bucketSpace) const; - void setDefaultDistribution(std::shared_ptr<lib::Distribution> distr); + void setDefaultDistribution(std::shared_ptr<const lib::Distribution> distr); + BucketSpaceMap::const_iterator begin() const { return _map.begin(); } + BucketSpaceMap::const_iterator end() const { return _map.end(); } + void add(document::BucketSpace bucketSpace, std::unique_ptr<DistributorBucketSpace> distributorBucketSpace); }; } diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributorcomponent.cpp index f8a0a5504ec..f0643eec37e 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp +++ b/storage/src/vespa/storage/distributor/distributorcomponent.cpp @@ -3,7 +3,8 @@ #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/log/log.h> LOG_SETUP(".distributorstoragelink"); @@ -137,11 +138,6 @@ DistributorComponent::nodeAddress(uint16_t nodeIndex) const nodeIndex); } -uint16_t -DistributorComponent::getRedundancy() const { - return getDistribution().getRedundancy(); -} - bool DistributorComponent::checkDistribution( api::StorageCommand &cmd, diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.h b/storage/src/vespa/storage/distributor/distributorcomponent.h index 307ddc20299..33e86d423e7 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.h +++ b/storage/src/vespa/storage/distributor/distributorcomponent.h @@ -89,11 +89,6 @@ public: bool storageNodeIsUp(uint32_t nodeIndex) const; /** - * Returns the current desired redundancy level. - */ - uint16_t getRedundancy() const; - - /** * Verifies that the given command has been received at the * correct distributor based on the current system state. */ @@ -157,12 +152,6 @@ public: return _distributor; } - virtual BucketDatabase& getBucketDatabase() = 0; - virtual const BucketDatabase& getBucketDatabase() const = 0; - // FIXME this hides the StorageComponent::getDistribution method, which - // even has a different signature altogether...! - virtual const lib::Distribution& getDistribution() const = 0; - DistributorBucketSpaceRepo &getBucketSpaceRepo() { return _bucketSpaceRepo; } const DistributorBucketSpaceRepo &getBucketSpaceRepo() const { return _bucketSpaceRepo; } diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index cd51387964a..bf27dc432b6 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -5,7 +5,6 @@ #include <vespa/storage/common/messagesender.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storage/distributor/maintenancebucket.h> #include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/bucketgctimecalculator.h> #include <vespa/storage/distributor/distributormetricsset.h> @@ -21,7 +20,6 @@ class DistributorInterface : public DistributorMessageSender { public: virtual PendingMessageTracker& getPendingMessageTracker() = 0; - virtual const lib::Distribution& getDistribution() const = 0; virtual DistributorMetricSet& getMetrics() = 0; diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index 77a86a3756d..902726fff41 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -20,6 +20,7 @@ #include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/stat.h> #include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/log/log.h> LOG_SETUP(".distributor.manager"); @@ -29,10 +30,9 @@ namespace storage::distributor { ExternalOperationHandler::ExternalOperationHandler( Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, const MaintenanceOperationGenerator& gen, DistributorComponentRegister& compReg) - : DistributorBucketSpaceComponent(owner, bucketSpaceRepo, bucketSpace, compReg, "External operation handler"), + : DistributorComponent(owner, bucketSpaceRepo, compReg, "External operation handler"), _operationGenerator(gen), _rejectFeedBeforeTimeReached() // At epoch { } diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 763796767cf..c405b63aa81 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -6,7 +6,6 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/distributor/distributorcomponent.h> -#include <vespa/storage/distributor/distributor_bucket_space_component.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <chrono> @@ -20,7 +19,7 @@ namespace distributor { class Distributor; class MaintenanceOperationGenerator; -class ExternalOperationHandler : public DistributorBucketSpaceComponent, +class ExternalOperationHandler : public DistributorComponent, public api::MessageHandler { public: @@ -39,7 +38,6 @@ public: ExternalOperationHandler(Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, const MaintenanceOperationGenerator&, DistributorComponentRegister& compReg); diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index 952f2f2f539..4ceeb387341 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -10,10 +10,13 @@ #include <vespa/storageapi/message/multioperation.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/log/log.h> LOG_SETUP(".distributor.operation.queue"); +using document::BucketSpace; using storage::lib::Node; using storage::lib::NodeType; @@ -23,12 +26,12 @@ namespace distributor { IdealStateManager::IdealStateManager( Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, DistributorComponentRegister& compReg, bool manageActiveBucketCopies) : HtmlStatusReporter("idealstateman", "Ideal state manager"), _metrics(new IdealStateMetricSet), - _distributorComponent(owner, bucketSpaceRepo, bucketSpace, compReg, "Ideal state manager") + _distributorComponent(owner, bucketSpaceRepo, compReg, "Ideal state manager"), + _bucketSpaceRepo(bucketSpaceRepo) { _distributorComponent.registerStatusPage(*this); _distributorComponent.registerMetric(*_metrics); @@ -74,17 +77,17 @@ IdealStateManager::iAmUp() const void IdealStateManager::fillParentAndChildBuckets(StateChecker::Context& c) const { - _distributorComponent.getBucketDatabase().getAll(c.bucketId, c.entries); + c.db.getAll(c.getBucketId(), c.entries); if (c.entries.empty()) { LOG(spam, "Did not find bucket %s in bucket database", - c.bucketId.toString().c_str()); + c.bucket.toString().c_str()); } } void IdealStateManager::fillSiblingBucket(StateChecker::Context& c) const { - c.siblingEntry = _distributorComponent.getBucketDatabase().get(c.siblingBucket); + c.siblingEntry = c.db.get(c.siblingBucket); } BucketDatabase::Entry* @@ -92,7 +95,7 @@ IdealStateManager::getEntryForPrimaryBucket(StateChecker::Context& c) const { for (uint32_t j = 0; j < c.entries.size(); ++j) { BucketDatabase::Entry& e = c.entries[j]; - if (e.getBucketId() == c.bucketId) { + if (e.getBucketId() == c.getBucketId()) { return &e; } } @@ -143,7 +146,8 @@ IdealStateManager::generateHighestPriority( const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const { - StateChecker::Context c(_distributorComponent, statsTracker, bucket.getBucketId()); + auto &distributorBucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); + StateChecker::Context c(_distributorComponent, distributorBucketSpace, statsTracker, bucket); fillParentAndChildBuckets(c); fillSiblingBucket(c); @@ -172,11 +176,14 @@ IdealStateManager::prioritize( } IdealStateOperation::SP -IdealStateManager::generateInterceptingSplit(const BucketDatabase::Entry& e, +IdealStateManager::generateInterceptingSplit(BucketSpace bucketSpace, + const BucketDatabase::Entry& e, api::StorageMessage::Priority pri) { NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(_distributorComponent, statsTracker, e.getBucketId()); + document::Bucket bucket(bucketSpace, e.getBucketId()); + auto &distributorBucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); + StateChecker::Context c(_distributorComponent, distributorBucketSpace, statsTracker, bucket); if (e.valid()) { c.entry = e; @@ -210,7 +217,8 @@ std::vector<MaintenanceOperation::SP> IdealStateManager::generateAll(const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const { - StateChecker::Context c(_distributorComponent, statsTracker, bucket.getBucketId()); + auto &distributorBucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); + StateChecker::Context c(_distributorComponent, distributorBucketSpace, statsTracker, bucket); fillParentAndChildBuckets(c); fillSiblingBucket(c); BucketDatabase::Entry* e(getEntryForPrimaryBucket(c)); @@ -233,7 +241,7 @@ IdealStateManager::generateAll(const document::Bucket &bucket, void IdealStateManager::getBucketStatus( - document::BucketSpace bucketSpace, + BucketSpace bucketSpace, const BucketDatabase::Entry& entry, NodeMaintenanceStatsTracker& statsTracker, std::ostream& out) const @@ -265,8 +273,10 @@ IdealStateManager::getBucketStatus( void IdealStateManager::getBucketStatus(std::ostream& out) const { - StatusBucketVisitor proc(*this, document::BucketSpace::placeHolder(), out); - _distributorComponent.getBucketDatabase().forEach(proc); + BucketSpace bucketSpace(BucketSpace::placeHolder()); + StatusBucketVisitor proc(*this, bucketSpace, out); + auto &distributorBucketSpace(_bucketSpaceRepo.get(bucketSpace)); + distributorBucketSpace.getBucketDatabase().forEach(proc); } } // distributor diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index b2ca5f8cef6..b9607b35d28 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -4,7 +4,7 @@ #include <deque> #include <map> #include <set> -#include <vespa/storage/distributor/distributor_bucket_space_component.h> +#include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/statechecker.h> #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> @@ -41,7 +41,6 @@ public: IdealStateManager(Distributor& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, - DistributorBucketSpace& bucketSpace, DistributorComponentRegister& compReg, bool manageActiveBucketCopies); @@ -68,6 +67,7 @@ public: * with higher priority than the given one. */ IdealStateOperation::SP generateInterceptingSplit( + document::BucketSpace bucketSpace, const BucketDatabase::Entry& e, api::StorageMessage::Priority pri); @@ -85,6 +85,8 @@ public: return _distributorComponent; } StorageComponent::LoadTypeSetSP getLoadTypes() { return _distributorComponent.getLoadTypes(); } + DistributorBucketSpaceRepo &getBucketSpaceRepo() { return _bucketSpaceRepo; } + const DistributorBucketSpaceRepo &getBucketSpaceRepo() const { return _bucketSpaceRepo; } private: void fillParentAndChildBuckets(StateChecker::Context& c) const; @@ -111,7 +113,8 @@ private: std::vector<StateChecker::SP> _stateCheckers; SplitBucketStateChecker* _splitBucketStateChecker; - DistributorBucketSpaceComponent _distributorComponent; + DistributorComponent _distributorComponent; + DistributorBucketSpaceRepo &_bucketSpaceRepo; std::vector<IdealStateOperation::SP> generateOperationsForBucket( StateChecker::Context& c) const; @@ -140,7 +143,6 @@ private: const BucketDatabase::Entry& entry, NodeMaintenanceStatsTracker& statsTracker, std::ostream& out) const; - }; } // distributor diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h index 783e8e1e5ba..c1d76b57c7c 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/document/bucket/bucketspace.h> #include <vespa/storage/bucketdb/bucketdatabase.h> namespace storage { @@ -13,20 +14,22 @@ public: class ScanResult { bool _done; + document::BucketSpace _bucketSpace; BucketDatabase::Entry _entry; public: bool isDone() const { return _done; } + document::BucketSpace getBucketSpace() const { return _bucketSpace; } const BucketDatabase::Entry& getEntry() const { return _entry; } static ScanResult createDone() { return ScanResult(true); } - static ScanResult createNotDone(BucketDatabase::Entry entry) { - return ScanResult(entry); + static ScanResult createNotDone(document::BucketSpace bucketSpace, BucketDatabase::Entry entry) { + return ScanResult(bucketSpace, entry); } private: - ScanResult(bool done) : _done(done), _entry() {} - ScanResult(const BucketDatabase::Entry& e) : _done(false), _entry(e) {} + ScanResult(bool done) : _done(done), _bucketSpace(document::BucketSpace::placeHolder()), _entry() {} + ScanResult(document::BucketSpace bucketSpace, const BucketDatabase::Entry& e) : _done(false), _bucketSpace(bucketSpace), _entry(e) {} }; virtual ScanResult scanNext() = 0; diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index 2bdef7ed320..870dcc25a4b 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -1,8 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simplemaintenancescanner.h" +#include <vespa/storage/distributor/distributor_bucket_space.h> namespace storage::distributor { +SimpleMaintenanceScanner::SimpleMaintenanceScanner(BucketPriorityDatabase& bucketPriorityDb, + const MaintenancePriorityGenerator& priorityGenerator, + const DistributorBucketSpaceRepo& bucketSpaceRepo) + : _bucketPriorityDb(bucketPriorityDb), + _priorityGenerator(priorityGenerator), + _bucketSpaceRepo(bucketSpaceRepo), + _bucketSpaceItr(_bucketSpaceRepo.begin()), + _bucketCursor() +{ +} + SimpleMaintenanceScanner::~SimpleMaintenanceScanner() {} SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() {} @@ -14,19 +26,28 @@ SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (const PendingMain MaintenanceScanner::ScanResult SimpleMaintenanceScanner::scanNext() { - BucketDatabase::Entry entry(_bucketDb.getNext(_bucketCursor)); - if (!entry.valid()) { - return ScanResult::createDone(); + for (;;) { + if (_bucketSpaceItr == _bucketSpaceRepo.end()) { + return ScanResult::createDone(); + } + const auto &bucketDb(_bucketSpaceItr->second->getBucketDatabase()); + BucketDatabase::Entry entry(bucketDb.getNext(_bucketCursor)); + if (!entry.valid()) { + ++_bucketSpaceItr; + _bucketCursor = document::BucketId(); + continue; + } + prioritizeBucket(document::Bucket(_bucketSpaceItr->first, entry.getBucketId())); + _bucketCursor = entry.getBucketId(); + return ScanResult::createNotDone(_bucketSpaceItr->first, entry); } - prioritizeBucket(document::Bucket(document::BucketSpace::placeHolder(), entry.getBucketId())); - _bucketCursor = entry.getBucketId(); - return ScanResult::createNotDone(entry); } void SimpleMaintenanceScanner::reset() { _bucketCursor = document::BucketId(); + _bucketSpaceItr = _bucketSpaceRepo.begin(); _pendingMaintenance = PendingMaintenanceStats(); } diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h index 05de7674d6a..f4ad53957e9 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h @@ -5,7 +5,7 @@ #include "bucketprioritydatabase.h" #include "maintenanceprioritygenerator.h" #include "node_maintenance_stats_tracker.h" -#include <vespa/storage/bucketdb/bucketdatabase.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> namespace storage { namespace distributor { @@ -31,18 +31,14 @@ public: private: BucketPriorityDatabase& _bucketPriorityDb; const MaintenancePriorityGenerator& _priorityGenerator; - const BucketDatabase& _bucketDb; + const DistributorBucketSpaceRepo &_bucketSpaceRepo; + DistributorBucketSpaceRepo::BucketSpaceMap::const_iterator _bucketSpaceItr; document::BucketId _bucketCursor; PendingMaintenanceStats _pendingMaintenance; public: SimpleMaintenanceScanner(BucketPriorityDatabase& bucketPriorityDb, const MaintenancePriorityGenerator& priorityGenerator, - const BucketDatabase& bucketDb) - : _bucketPriorityDb(bucketPriorityDb), - _priorityGenerator(priorityGenerator), - _bucketDb(bucketDb), - _bucketCursor() - {} + const DistributorBucketSpaceRepo& bucketSpaceRepo); SimpleMaintenanceScanner(const SimpleMaintenanceScanner&) = delete; SimpleMaintenanceScanner& operator=(const SimpleMaintenanceScanner&) = delete; ~SimpleMaintenanceScanner(); diff --git a/storage/src/vespa/storage/distributor/maintenancebucket.h b/storage/src/vespa/storage/distributor/maintenancebucket.h deleted file mode 100644 index a44381830c5..00000000000 --- a/storage/src/vespa/storage/distributor/maintenancebucket.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/document/bucket/bucketid.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/storage/distributor/maintenance/maintenancepriority.h> - -namespace storage { - -namespace distributor { - -/** - * Simple container to communicate a bucket that needs to be - * checked for maintenanceoperations. - */ -class MaintenanceBucket { -public: - typedef MaintenancePriority::Priority Priority; - - MaintenanceBucket() - : node(0), - pri(MaintenancePriority::NO_MAINTENANCE_NEEDED) - {} - - MaintenanceBucket(const document::BucketId& bid_, - uint16_t node_, - Priority pri_) - : bid(bid_), - node(node_), - pri(pri_) - { - - } - - // The bucket to be checked. - document::BucketId bid; - - // The primary node of the bucket. - uint16_t node; - - // The priority to check the bucket. - Priority pri; - - bool requiresMaintenance() const { - return pri != MaintenancePriority::NO_MAINTENANCE_NEEDED; - } - - std::string toString() const { - return vespalib::make_string("MaintenanceBucket(%s: Node %d, Pri %s)", - bid.toString().c_str(), - (int)node, - MaintenancePriority::toString(pri).c_str()); - } -}; - -} - -} - diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 7ef03cb696a..659a7f1d435 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -283,7 +283,8 @@ PutOperation::onStart(DistributorMessageSender& sender) _bucketSpace.getBucketDatabase(), idealNodeCalculator, _manager.getDistributor().getConfig().getMinimalBucketSplit(), - _bucketSpace.getDistribution().getRedundancy()); + _bucketSpace.getDistribution().getRedundancy(), + _msg->getBucket().getBucketSpace()); OperationTargetList targets(targetResolver.getTargets( OperationTargetResolver::PUT, bid)); diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp index a4cebcc7c3e..9f92d313f1f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp @@ -3,6 +3,7 @@ #include "visitoroperation.h" #include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/bucketownership.h> #include <vespa/storage/distributor/operations/external/visitororder.h> #include <vespa/storage/distributor/visitormetricsset.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index d78262709e3..53d9cc018f9 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -3,6 +3,7 @@ #include "garbagecollectionoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/log/log.h> @@ -21,7 +22,7 @@ GarbageCollectionOperation::~GarbageCollectionOperation() { } void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) { - BucketDatabase::Entry entry = _manager->getDistributorComponent().getBucketDatabase().get(getBucketId()); + BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); std::vector<uint16_t> nodes = entry->getNodes(); for (uint32_t i = 0; i < nodes.size(); i++) { @@ -62,11 +63,11 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, if (_tracker.finished()) { if (_ok) { - BucketDatabase::Entry dbentry = _manager->getDistributorComponent().getBucketDatabase().get(getBucketId()); + BucketDatabase::Entry dbentry = _bucketSpace->getBucketDatabase().get(getBucketId()); if (dbentry.valid()) { dbentry->setLastGarbageCollectionTime( _manager->getDistributorComponent().getClock().getTimeInSeconds().getTime()); - _manager->getDistributorComponent().getBucketDatabase().update(dbentry); + _bucketSpace->getBucketDatabase().update(dbentry); } } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index 3095dce7b87..2337129e375 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/distributor/pendingmessagetracker.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/log/log.h> @@ -25,7 +26,8 @@ const uint32_t IdealStateOperation::MAINTENANCE_MESSAGE_TYPES[] = }; IdealStateOperation::IdealStateOperation(const BucketAndNodes& bucketAndNodes) - : _manager(NULL), + : _manager(nullptr), + _bucketSpace(nullptr), _bucketAndNodes(bucketAndNodes), _ok(true), _priority(255) @@ -78,6 +80,12 @@ BucketAndNodes::toString() const } void +IdealStateOperation::setIdealStateManager(IdealStateManager* manager) { + _manager = manager; + _bucketSpace = &_manager->getBucketSpaceRepo().get(getBucket().getBucketSpace()); +}; + +void IdealStateOperation::done() { if (_manager != NULL) { @@ -188,19 +196,10 @@ checkNullBucketRequestBucketInfoMessage(uint16_t node, const PendingMessageTracker& tracker) { RequestBucketInfoChecker rchk; - for (;;) { - // Check messages sent to null-bucket (i.e. any bucket) for the node. - document::Bucket nullBucket(bucketSpace, document::BucketId()); - tracker.checkPendingMessages(node, nullBucket, rchk); - if (rchk.blocked) { - return true; - } - if (bucketSpace == BucketSpace::placeHolder()) { - break; - } - bucketSpace = BucketSpace::placeHolder(); - } - return false; + // Check messages sent to null-bucket (i.e. any bucket) for the node. + document::Bucket nullBucket(bucketSpace, document::BucketId()); + tracker.checkPendingMessages(node, nullBucket, rchk); + return rchk.blocked; } } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index 9824ae0630f..e8480902549 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -10,6 +10,7 @@ namespace storage::distributor { +class DistributorBucketSpace; class PendingMessageTracker; class IdealStateManager; @@ -165,9 +166,7 @@ public: @param manager The ideal state manager. */ - void setIdealStateManager(IdealStateManager* manager) { - _manager = manager; - }; + void setIdealStateManager(IdealStateManager* manager); /** Returns the type of operation this is. @@ -224,6 +223,7 @@ protected: friend class IdealStateManager; IdealStateManager* _manager; + DistributorBucketSpace *_bucketSpace; BucketAndNodes _bucketAndNodes; std::string _detailedReason; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp index 77135f56399..52a4a5c195c 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp @@ -2,6 +2,7 @@ #include "joinoperation.h" #include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <climits> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.join"); @@ -42,7 +43,7 @@ JoinOperation::NodeToBuckets JoinOperation::resolveSourceBucketsPerTargetNode() const { NodeToBuckets nodeToBuckets; - const auto& db(_manager->getDistributorComponent().getBucketDatabase()); + const auto& db(_bucketSpace->getBucketDatabase()); for (const auto& bucket : _bucketsToJoin) { BucketDatabase::Entry entry(db.get(bucket)); @@ -117,7 +118,7 @@ JoinOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP& LOG(spam, "Adding joined bucket %s", getBucketId().toString().c_str()); } } else if (rep.getResult().getResult() == api::ReturnCode::BUCKET_NOT_FOUND - && _manager->getDistributorComponent().getBucketDatabase().get(getBucketId())->getNode(node) != 0) + && _bucketSpace->getBucketDatabase().get(getBucketId())->getNode(node) != 0) { _manager->getDistributorComponent().recheckBucketInfo(node, getBucket()); LOGBP(warning, "Join failed to find %s: %s", diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index e889dbe279b..271ac35968e 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mergeoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <array> #include <vespa/log/bufferedlogger.h> @@ -104,7 +105,7 @@ struct NodeIndexComparator void MergeOperation::onStart(DistributorMessageSender& sender) { - BucketDatabase::Entry entry = _manager->getDistributorComponent().getBucketDatabase().get(getBucketId()); + BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); if (!entry.valid()) { LOGBP(debug, "Unable to merge nonexisting bucket %s", getBucketId().toString().c_str()); _ok = false; @@ -126,7 +127,7 @@ MergeOperation::onStart(DistributorMessageSender& sender) } _infoBefore = entry.getBucketInfo(); - generateSortedNodeList(_manager->getDistributorComponent().getDistribution(), + generateSortedNodeList(_bucketSpace->getDistribution(), clusterState, getBucketId(), _limiter, @@ -273,7 +274,7 @@ MergeOperation::onReceive(DistributorMessageSender& sender, _ok = result.success(); if (_ok) { BucketDatabase::Entry entry( - _manager->getDistributorComponent().getBucketDatabase().get(getBucketId())); + _bucketSpace->getBucketDatabase().get(getBucketId())); if (!entry.valid()) { LOG(debug, "Bucket %s no longer exists after merge", getBucketId().toString().c_str()); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index 6c0245cb590..9a94a5a62ad 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -3,6 +3,7 @@ #include "removebucketoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/log/log.h> @@ -15,7 +16,7 @@ RemoveBucketOperation::onStartInternal(DistributorMessageSender& sender) { std::vector<std::pair<uint16_t, std::shared_ptr<api::DeleteBucketCommand> > > msgs; - BucketDatabase::Entry entry = _manager->getDistributorComponent().getBucketDatabase().get(getBucketId()); + BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); for (uint32_t i = 0; i < getNodes().size(); ++i) { uint16_t node = getNodes()[i]; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp index f3528d30aba..1acb2dcc64b 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp @@ -2,6 +2,7 @@ #include "setbucketstateoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/log/log.h> LOG_SETUP(".distributor.operation.idealstate.setactive"); @@ -83,7 +84,7 @@ SetBucketStateOperation::onReceive(DistributorMessageSender& sender, bool deactivate = false; if (reply->getResult().success()) { BucketDatabase::Entry entry = - _manager->getDistributorComponent().getBucketDatabase().get(rep.getBucketId()); + _bucketSpace->getBucketDatabase().get(rep.getBucketId()); if (entry.valid()) { const BucketCopy* copy = entry->getNode(node); @@ -103,7 +104,7 @@ SetBucketStateOperation::onReceive(DistributorMessageSender& sender, node, bInfo).setTrusted(copy->trusted())); - _manager->getDistributorComponent().getBucketDatabase().update(entry); + _bucketSpace->getBucketDatabase().update(entry); } } else { LOG(debug, "%s did not exist when receiving %s", diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index a8f547afe45..1b40f744a80 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <climits> #include <vespa/log/bufferedlogger.h> @@ -26,8 +27,7 @@ SplitOperation::onStart(DistributorMessageSender& sender) { _ok = false; - BucketDatabase::Entry entry = _manager->getDistributorComponent() - .getBucketDatabase().get(getBucketId()); + BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); for (uint32_t i = 0; i < entry->getNodeCount(); i++) { std::shared_ptr<api::SplitBucketCommand> msg( @@ -66,7 +66,7 @@ SplitOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP if (rep.getResult().success()) { BucketDatabase::Entry entry = - _manager->getDistributorComponent().getBucketDatabase().get(rep.getBucketId()); + _bucketSpace->getBucketDatabase().get(rep.getBucketId()); if (entry.valid()) { entry->removeNode(node); @@ -74,9 +74,9 @@ SplitOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP if (entry->getNodeCount() == 0) { LOG(spam, "Removing split bucket %s", getBucketId().toString().c_str()); - _manager->getDistributorComponent().getBucketDatabase().remove(rep.getBucketId()); + _bucketSpace->getBucketDatabase().remove(rep.getBucketId()); } else { - _manager->getDistributorComponent().getBucketDatabase().update(entry); + _bucketSpace->getBucketDatabase().update(entry); } ost << getBucketId() << " => "; @@ -115,7 +115,7 @@ SplitOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP } } else if ( rep.getResult().getResult() == api::ReturnCode::BUCKET_NOT_FOUND - && _manager->getDistributorComponent().getBucketDatabase().get(rep.getBucketId())->getNode(node) != 0) + && _bucketSpace->getBucketDatabase().get(rep.getBucketId())->getNode(node) != 0) { _manager->getDistributorComponent().recheckBucketInfo(node, getBucket()); LOGBP(debug, "Split failed for %s: bucket not found. Storage and " diff --git a/storage/src/vespa/storage/distributor/operationtargetresolver.cpp b/storage/src/vespa/storage/distributor/operationtargetresolver.cpp index f537c6b67b6..fa4697f7861 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolver.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolver.cpp @@ -6,15 +6,9 @@ namespace storage { namespace distributor { -document::Bucket -OperationTarget::getBucket() const -{ - return document::Bucket(document::BucketSpace::placeHolder(), _bucket); -} - void OperationTarget::print(vespalib::asciistream& out, const PrintProperties&) const { - out << "OperationTarget(" << _bucket << ", " << _node + out << "OperationTarget(" << _bucket.toString() << ", " << _node << (_newCopy ? ", new copy" : ", existing copy") << ")"; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolver.h b/storage/src/vespa/storage/distributor/operationtargetresolver.h index b9f7537b5f5..23e0fbbcba4 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolver.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolver.h @@ -15,17 +15,17 @@ namespace distributor { class OperationTarget : public vespalib::AsciiPrintable { - document::BucketId _bucket; + document::Bucket _bucket; lib::Node _node; bool _newCopy; public: OperationTarget() : _newCopy(true) {} - OperationTarget(const document::BucketId& id, const lib::Node& node, bool newCopy) - : _bucket(id), _node(node), _newCopy(newCopy) {} + OperationTarget(const document::Bucket& bucket, const lib::Node& node, bool newCopy) + : _bucket(bucket), _node(node), _newCopy(newCopy) {} - const document::BucketId& getBucketId() const { return _bucket; } - document::Bucket getBucket() const; + document::BucketId getBucketId() const { return _bucket.getBucketId(); } + document::Bucket getBucket() const { return _bucket; } const lib::Node& getNode() const { return _node; } bool isNewCopy() const { return _newCopy; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index 942ec3705bf..23bb6b1db78 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -129,12 +129,12 @@ BucketInstanceList::extendToEnoughCopies( } OperationTargetList -BucketInstanceList::createTargets() +BucketInstanceList::createTargets(document::BucketSpace bucketSpace) { OperationTargetList result; for (uint32_t i=0; i<_instances.size(); ++i) { BucketInstance& bi(_instances[i]); - result.push_back(OperationTarget(bi._bucket, bi._node, !bi._exist)); + result.push_back(OperationTarget(document::Bucket(bucketSpace, bi._bucket), bi._node, !bi._exist)); } return result; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 2f4a3e0117a..73a2c281b18 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -71,7 +71,7 @@ public: std::sort(_instances.begin(), _instances.end(), order); } - OperationTargetList createTargets(); + OperationTargetList createTargets(document::BucketSpace bucketSpace); void print(vespalib::asciistream& out, const PrintProperties& p) const override; }; @@ -81,16 +81,19 @@ class OperationTargetResolverImpl : public OperationTargetResolver { const lib::IdealNodeCalculator& _idealNodeCalculator; uint32_t _minUsedBucketBits; uint16_t _redundancy; + document::BucketSpace _bucketSpace; public: OperationTargetResolverImpl(BucketDatabase& bucketDatabase, const lib::IdealNodeCalculator& idealNodeCalc, uint32_t minUsedBucketBits, - uint16_t redundancy) + uint16_t redundancy, + document::BucketSpace bucketSpace) : _bucketDatabase(bucketDatabase), _idealNodeCalculator(idealNodeCalc), _minUsedBucketBits(minUsedBucketBits), - _redundancy(redundancy) + _redundancy(redundancy), + _bucketSpace(bucketSpace) {} BucketInstanceList getAllInstances(OperationType type, @@ -102,7 +105,7 @@ public: } OperationTargetList getTargets(OperationType type, const document::BucketId& id) override { - return getInstances(type, id).createTargets(); + return getInstances(type, id).createTargets(_bucketSpace); } }; diff --git a/storage/src/vespa/storage/distributor/outdated_nodes.h b/storage/src/vespa/storage/distributor/outdated_nodes.h new file mode 100644 index 00000000000..fddb1806d82 --- /dev/null +++ b/storage/src/vespa/storage/distributor/outdated_nodes.h @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <unordered_set> + +namespace storage::distributor::dbtransition { + +using OutdatedNodes = std::unordered_set<uint16_t>; + +} diff --git a/storage/src/vespa/storage/distributor/outdated_nodes_map.h b/storage/src/vespa/storage/distributor/outdated_nodes_map.h new file mode 100644 index 00000000000..8d08b20732b --- /dev/null +++ b/storage/src/vespa/storage/distributor/outdated_nodes_map.h @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "outdated_nodes.h" +#include <vespa/document/bucket/bucketspace.h> +#include <unordered_map> + +namespace storage::distributor::dbtransition { + +using OutdatedNodesMap = std::unordered_map<document::BucketSpace, OutdatedNodes, document::BucketSpace::hash>; + +} diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp new file mode 100644 index 00000000000..ed9c8bc222b --- /dev/null +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp @@ -0,0 +1,421 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "pending_bucket_space_db_transition.h" +#include "clusterinformation.h" +#include "pendingclusterstate.h" +#include "distributor_bucket_space.h" +#include <vespa/storage/common/bucketoperationlogger.h> +#include <algorithm> + +#include <vespa/log/log.h> +LOG_SETUP(".pendingbucketspacedbtransition"); + +namespace storage::distributor { + +using lib::Node; +using lib::NodeType; +using lib::NodeState; + +PendingBucketSpaceDbTransition::PendingBucketSpaceDbTransition(const PendingClusterState &pendingClusterState, + DistributorBucketSpace &distributorBucketSpace, + bool distributionChanged, + const OutdatedNodes &outdatedNodes, + std::shared_ptr<const ClusterInformation> clusterInfo, + const lib::ClusterState &newClusterState, + api::Timestamp creationTimestamp) + : _entries(), + _iter(0), + _removedBuckets(), + _missingEntries(), + _clusterInfo(std::move(clusterInfo)), + _outdatedNodes(newClusterState.getNodeCount(NodeType::STORAGE)), + _prevClusterState(_clusterInfo->getClusterState()), + _newClusterState(newClusterState), + _creationTimestamp(creationTimestamp), + _pendingClusterState(pendingClusterState), + _distributorBucketSpace(distributorBucketSpace), + _distributorIndex(_clusterInfo->getDistributorIndex()), + _bucketOwnershipTransfer(distributionChanged) +{ + if (distributorChanged()) { + _bucketOwnershipTransfer = true; + } + if (_bucketOwnershipTransfer) { + markAllAvailableNodesAsRequiringRequest(); + } else { + updateSetOfNodesThatAreOutdated(); + addAdditionalNodesToOutdatedSet(outdatedNodes); + } +} + +PendingBucketSpaceDbTransition::~PendingBucketSpaceDbTransition() +{ +} + +PendingBucketSpaceDbTransition::Range +PendingBucketSpaceDbTransition::skipAllForSameBucket() +{ + Range r(_iter, _iter); + + for (document::BucketId& bid = _entries[_iter].bucketId; + _iter < _entries.size() && _entries[_iter].bucketId == bid; + ++_iter) + { + } + + r.second = _iter; + return r; +} + +std::vector<BucketCopy> +PendingBucketSpaceDbTransition::getCopiesThatAreNewOrAltered(BucketDatabase::Entry& info, const Range& range) +{ + std::vector<BucketCopy> copiesToAdd; + for (uint32_t i = range.first; i < range.second; ++i) { + const BucketCopy& candidate(_entries[i].copy); + const BucketCopy* cp = info->getNode(candidate.getNode()); + + if (!cp || !(cp->getBucketInfo() == candidate.getBucketInfo())) { + copiesToAdd.push_back(candidate); + } + } + return copiesToAdd; +} + +void +PendingBucketSpaceDbTransition::insertInfo(BucketDatabase::Entry& info, const Range& range) +{ + std::vector<BucketCopy> copiesToAddOrUpdate( + getCopiesThatAreNewOrAltered(info, range)); + + const auto &dist(_distributorBucketSpace.getDistribution()); + std::vector<uint16_t> order( + dist.getIdealStorageNodes( + _newClusterState, + _entries[range.first].bucketId, + _clusterInfo->getStorageUpStates())); + info->addNodes(copiesToAddOrUpdate, order, TrustedUpdate::DEFER); + + LOG_BUCKET_OPERATION_NO_LOCK( + _entries[range.first].bucketId, + vespalib::make_string("insertInfo: %s", + info.toString().c_str())); +} + +std::string +PendingBucketSpaceDbTransition::requestNodesToString() +{ + return _pendingClusterState.requestNodesToString(); +} + +bool +PendingBucketSpaceDbTransition::removeCopiesFromNodesThatWereRequested(BucketDatabase::Entry& e, const document::BucketId& bucketId) +{ + bool updated = false; + for (uint32_t i = 0; i < e->getNodeCount();) { + auto& info(e->getNodeRef(i)); + const uint16_t entryNode(info.getNode()); + // Don't remove an entry if it's been updated in the time after the + // bucket info requests were sent, as this would erase newer state. + // Don't immediately update trusted state, as that could erroneously + // mark a single remaining replica as trusted even though there might + // be one or more additional replicas pending merge into the database. + if (nodeIsOutdated(entryNode) + && (info.getTimestamp() < _creationTimestamp) + && e->removeNode(entryNode, TrustedUpdate::DEFER)) + { + LOG(spam, + "Removed bucket %s from node %d", + bucketId.toString().c_str(), + entryNode); + updated = true; + // After removing current node, getNodeRef(i) will point to the _next_ node, so don't increment `i`. + } else { + ++i; + } + } + return updated; +} + +bool +PendingBucketSpaceDbTransition::databaseIteratorHasPassedBucketInfoIterator(const document::BucketId& bucketId) const +{ + return (_iter < _entries.size() + && _entries[_iter].bucketId.toKey() < bucketId.toKey()); +} + +bool +PendingBucketSpaceDbTransition::bucketInfoIteratorPointsToBucket(const document::BucketId& bucketId) const +{ + return _iter < _entries.size() && _entries[_iter].bucketId == bucketId; +} + +bool +PendingBucketSpaceDbTransition::process(BucketDatabase::Entry& e) +{ + document::BucketId bucketId(e.getBucketId()); + + LOG(spam, + "Before merging info from nodes [%s], bucket %s had info %s", + requestNodesToString().c_str(), + bucketId.toString().c_str(), + e.getBucketInfo().toString().c_str()); + + while (databaseIteratorHasPassedBucketInfoIterator(bucketId)) { + LOG(spam, "Found new bucket %s, adding", + _entries[_iter].bucketId.toString().c_str()); + + _missingEntries.push_back(skipAllForSameBucket()); + } + + bool updated(removeCopiesFromNodesThatWereRequested(e, bucketId)); + + if (bucketInfoIteratorPointsToBucket(bucketId)) { + LOG(spam, "Updating bucket %s", + _entries[_iter].bucketId.toString().c_str()); + + insertInfo(e, skipAllForSameBucket()); + updated = true; + } + + if (updated) { + // Remove bucket if we've previously removed all nodes from it + if (e->getNodeCount() == 0) { + _removedBuckets.push_back(bucketId); + } else { + e.getBucketInfo().updateTrusted(); + } + } + + LOG(spam, + "After merging info from nodes [%s], bucket %s had info %s", + requestNodesToString().c_str(), + bucketId.toString().c_str(), + e.getBucketInfo().toString().c_str()); + + return true; +} + +void +PendingBucketSpaceDbTransition::addToBucketDB(BucketDatabase& db, const Range& range) +{ + LOG(spam, "Adding new bucket %s with %d copies", + _entries[range.first].bucketId.toString().c_str(), + range.second - range.first); + + BucketDatabase::Entry e(_entries[range.first].bucketId, BucketInfo()); + insertInfo(e, range); + if (e->getLastGarbageCollectionTime() == 0) { + e->setLastGarbageCollectionTime( + framework::MicroSecTime(_creationTimestamp) + .getSeconds().getTime()); + } + e.getBucketInfo().updateTrusted(); + db.update(e); +} + +void +PendingBucketSpaceDbTransition::mergeIntoBucketDatabase() +{ + BucketDatabase &db(_distributorBucketSpace.getBucketDatabase()); + std::sort(_entries.begin(), _entries.end()); + + db.forEach(*this); + + for (uint32_t i = 0; i < _removedBuckets.size(); ++i) { + db.remove(_removedBuckets[i]); + } + _removedBuckets.clear(); + + // All of the remaining were not already in the bucket database. + while (_iter < _entries.size()) { + _missingEntries.push_back(skipAllForSameBucket()); + } + + for (uint32_t i = 0; i < _missingEntries.size(); ++i) { + addToBucketDB(db, _missingEntries[i]); + } +} + +void +PendingBucketSpaceDbTransition::onRequestBucketInfoReply(const api::RequestBucketInfoReply &reply, uint16_t node) +{ + for (const auto &entry : reply.getBucketInfo()) { + _entries.emplace_back(entry._bucketId, + BucketCopy(_creationTimestamp, + node, + entry._info)); + } +} + +bool +PendingBucketSpaceDbTransition::distributorChanged() +{ + const auto &oldState(_prevClusterState); + const auto &newState(_newClusterState); + if (newState.getDistributionBitCount() != oldState.getDistributionBitCount()) { + return true; + } + + Node myNode(NodeType::DISTRIBUTOR, _distributorIndex); + if (oldState.getNodeState(myNode).getState() == lib::State::DOWN) { + return true; + } + + uint16_t oldCount = oldState.getNodeCount(NodeType::DISTRIBUTOR); + uint16_t newCount = newState.getNodeCount(NodeType::DISTRIBUTOR); + + uint16_t maxCount = std::max(oldCount, newCount); + + for (uint16_t i = 0; i < maxCount; ++i) { + Node node(NodeType::DISTRIBUTOR, i); + + const lib::State& old(oldState.getNodeState(node).getState()); + const lib::State& nw(newState.getNodeState(node).getState()); + + if (nodeWasUpButNowIsDown(old, nw)) { + if (nodeInSameGroupAsSelf(i) || + nodeNeedsOwnershipTransferFromGroupDown(i, newState)) { + return true; + } + } + } + + return false; +} + +bool +PendingBucketSpaceDbTransition::nodeWasUpButNowIsDown(const lib::State& old, + const lib::State& nw) +{ + return (old.oneOf("uimr") && !nw.oneOf("uimr")); +} + +bool +PendingBucketSpaceDbTransition::nodeInSameGroupAsSelf(uint16_t index) const +{ + const auto &dist(_distributorBucketSpace.getDistribution()); + if (dist.getNodeGraph().getGroupForNode(index) == + dist.getNodeGraph().getGroupForNode(_distributorIndex)) { + LOG(debug, + "Distributor %d state changed, need to request data from all " + "storage nodes", + index); + return true; + } else { + LOG(debug, + "Distributor %d state changed but unrelated to my group.", + index); + return false; + } +} + +bool +PendingBucketSpaceDbTransition::nodeNeedsOwnershipTransferFromGroupDown( + uint16_t nodeIndex, + const lib::ClusterState& state) const +{ + const auto &dist(_distributorBucketSpace.getDistribution()); + if (!dist.distributorAutoOwnershipTransferOnWholeGroupDown()) { + return false; // Not doing anything for downed groups. + } + const lib::Group* group(dist.getNodeGraph().getGroupForNode(nodeIndex)); + // If there is no group information associated with the node (because the + // group has changed or the node has been removed from config), we must + // also invoke ownership transfer of buckets. + if (group == nullptr + || lib::Distribution::allDistributorsDown(*group, state)) + { + LOG(debug, + "Distributor %u state changed and is in a " + "group that now has no distributors remaining", + nodeIndex); + return true; + } + return false; +} + +uint16_t +PendingBucketSpaceDbTransition::newStateStorageNodeCount() const +{ + return _newClusterState.getNodeCount(lib::NodeType::STORAGE); +} + +bool +PendingBucketSpaceDbTransition::storageNodeMayHaveLostData(uint16_t index) +{ + Node node(NodeType::STORAGE, index); + NodeState newState = _newClusterState.getNodeState(node); + NodeState oldState = _prevClusterState.getNodeState(node); + + return (newState.getStartTimestamp() > oldState.getStartTimestamp()); +} + +void +PendingBucketSpaceDbTransition::updateSetOfNodesThatAreOutdated() +{ + const uint16_t nodeCount(newStateStorageNodeCount()); + for (uint16_t index = 0; index < nodeCount; ++index) { + if (storageNodeMayHaveLostData(index) || storageNodeChanged(index)) { + _outdatedNodes.insert(index); + } + } +} + +bool +PendingBucketSpaceDbTransition::storageNodeChanged(uint16_t index) { + Node node(NodeType::STORAGE, index); + NodeState newState = _newClusterState.getNodeState(node); + NodeState oldNodeState = _prevClusterState.getNodeState(node); + + // similarTo() also covers disk states. + if (!(oldNodeState.similarTo(newState))) { + LOG(debug, + "State for storage node %d has changed from '%s' to '%s', " + "updating bucket information", + index, + oldNodeState.toString().c_str(), + newState.toString().c_str()); + return true; + } + + return false; +} + +bool +PendingBucketSpaceDbTransition::storageNodeUpInNewState(uint16_t node) const +{ + return _newClusterState.getNodeState(Node(NodeType::STORAGE, node)) + .getState().oneOf(_clusterInfo->getStorageUpStates()); +} + +void +PendingBucketSpaceDbTransition::markAllAvailableNodesAsRequiringRequest() +{ + const uint16_t nodeCount(newStateStorageNodeCount()); + for (uint16_t i = 0; i < nodeCount; ++i) { + if (storageNodeUpInNewState(i)) { + _outdatedNodes.insert(i); + } + } +} + +void +PendingBucketSpaceDbTransition::addAdditionalNodesToOutdatedSet( + const std::unordered_set<uint16_t>& nodes) +{ + const uint16_t nodeCount(newStateStorageNodeCount()); + for (uint16_t node : nodes) { + if (node < nodeCount) { + _outdatedNodes.insert(node); + } + } +} + +void +PendingBucketSpaceDbTransition::addNodeInfo(const document::BucketId& id, const BucketCopy& copy) +{ + _entries.emplace_back(id, copy); +} + +} diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h new file mode 100644 index 00000000000..903f9b762fb --- /dev/null +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h @@ -0,0 +1,116 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "pending_bucket_space_db_transition_entry.h" +#include "outdated_nodes.h" +#include <vespa/storage/bucketdb/bucketdatabase.h> + +namespace storage::api { class RequestBucketInfoReply; } +namespace storage::lib { class ClusterState; class State; } + +namespace storage::distributor { + +class ClusterInformation; +class PendingClusterState; +class DistributorBucketSpace; + +/** + * Class used by PendingClusterState to track request bucket info + * reply result within a bucket space and apply it to the distributor + * bucket database when switching to the pending cluster state. + */ +class PendingBucketSpaceDbTransition : public BucketDatabase::MutableEntryProcessor +{ +public: + using Entry = dbtransition::Entry; + using EntryList = std::vector<Entry>; + using OutdatedNodes = dbtransition::OutdatedNodes; +private: + using Range = std::pair<uint32_t, uint32_t>; + + EntryList _entries; + uint32_t _iter; + std::vector<document::BucketId> _removedBuckets; + std::vector<Range> _missingEntries; + std::shared_ptr<const ClusterInformation> _clusterInfo; + + // Set for all nodes that may have changed state since that previous + // active cluster state, or that were marked as outdated when the pending + // cluster state was constructed. + // May be a superset of _requestedNodes, as some nodes that are outdated + // may be down and thus cannot get a request. + OutdatedNodes _outdatedNodes; + + const lib::ClusterState &_prevClusterState; + const lib::ClusterState &_newClusterState; + const api::Timestamp _creationTimestamp; + const PendingClusterState &_pendingClusterState; + DistributorBucketSpace &_distributorBucketSpace; + uint16_t _distributorIndex; + bool _bucketOwnershipTransfer; + + // BucketDataBase::MutableEntryProcessor API + bool process(BucketDatabase::Entry& e) override; + + /** + * Skips through all entries for the same bucket and returns + * the range in the entry list for which they were found. + * The range is [from, to> + */ + Range skipAllForSameBucket(); + + std::vector<BucketCopy> getCopiesThatAreNewOrAltered(BucketDatabase::Entry& info, const Range& range); + void insertInfo(BucketDatabase::Entry& info, const Range& range); + void addToBucketDB(BucketDatabase& db, const Range& range); + + bool nodeIsOutdated(uint16_t node) const { + return (_outdatedNodes.find(node) != _outdatedNodes.end()); + } + + // Returns whether at least one replica was removed from the entry. + // Does NOT implicitly update trusted status on remaining replicas; caller must do + // this explicitly. + bool removeCopiesFromNodesThatWereRequested(BucketDatabase::Entry& e, const document::BucketId& bucketId); + + // Helper methods for iterating over _entries + bool databaseIteratorHasPassedBucketInfoIterator(const document::BucketId& bucketId) const; + bool bucketInfoIteratorPointsToBucket(const document::BucketId& bucketId) const; + std::string requestNodesToString(); + + bool distributorChanged(); + static bool nodeWasUpButNowIsDown(const lib::State &old, const lib::State &nw); + bool storageNodeUpInNewState(uint16_t node) const; + bool nodeInSameGroupAsSelf(uint16_t index) const; + bool nodeNeedsOwnershipTransferFromGroupDown(uint16_t nodeIndex, const lib::ClusterState& state) const; + uint16_t newStateStorageNodeCount() const; + bool storageNodeMayHaveLostData(uint16_t index); + bool storageNodeChanged(uint16_t index); + void markAllAvailableNodesAsRequiringRequest(); + void addAdditionalNodesToOutdatedSet(const OutdatedNodes &nodes); + void updateSetOfNodesThatAreOutdated(); + +public: + PendingBucketSpaceDbTransition(const PendingClusterState &pendingClusterState, + DistributorBucketSpace &distributorBucketSpace, + bool distributionChanged, + const OutdatedNodes &outdatedNodes, + std::shared_ptr<const ClusterInformation> clusterInfo, + const lib::ClusterState &newClusterState, + api::Timestamp creationTimestamp); + ~PendingBucketSpaceDbTransition(); + + // Merges all the results with the corresponding bucket database. + void mergeIntoBucketDatabase(); + + // Adds the info from the reply to our list of information. + void onRequestBucketInfoReply(const api::RequestBucketInfoReply &reply, uint16_t node); + + const OutdatedNodes &getOutdatedNodes() { return _outdatedNodes; } + bool getBucketOwnershipTransfer() const { return _bucketOwnershipTransfer; } + + // Methods used by unit tests. + const EntryList& results() const { return _entries; } + void addNodeInfo(const document::BucketId& id, const BucketCopy& copy); +}; + +} diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h new file mode 100644 index 00000000000..ad6e0695be1 --- /dev/null +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/storage/bucketdb/bucketcopy.h> + +namespace storage::distributor::dbtransition { + +struct Entry { + Entry(const document::BucketId& bid, + const BucketCopy& copy_) + : bucketId(bid), + copy(copy_) + {} + + document::BucketId bucketId; + BucketCopy copy; + + bool operator<(const Entry& other) const { + return bucketId.toKey() < other.bucketId.toKey(); + } +}; + +} diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index 9ad803b7b7f..71684db5527 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "pendingclusterstate.h" +#include "pending_bucket_space_db_transition.h" #include "bucketdbupdater.h" +#include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/vespalib/util/xmlstream.hpp> @@ -22,68 +25,82 @@ PendingClusterState::PendingClusterState( const framework::Clock& clock, const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, + DistributorBucketSpaceRepo &bucketSpaceRepo, const std::shared_ptr<api::SetSystemStateCommand>& newStateCmd, - const std::unordered_set<uint16_t>& outdatedNodes, + const OutdatedNodesMap &outdatedNodesMap, api::Timestamp creationTimestamp) : _cmd(newStateCmd), _requestedNodes(newStateCmd->getSystemState().getNodeCount(lib::NodeType::STORAGE)), - _outdatedNodes(newStateCmd->getSystemState().getNodeCount(lib::NodeType::STORAGE)), - _iter(0), _prevClusterState(clusterInfo->getClusterState()), _newClusterState(newStateCmd->getSystemState()), _clock(clock), _clusterInfo(clusterInfo), _creationTimestamp(creationTimestamp), _sender(sender), - _bucketOwnershipTransfer(distributorChanged(_prevClusterState, _newClusterState)) + _bucketSpaceRepo(bucketSpaceRepo), + _bucketOwnershipTransfer(false), + _pendingTransitions() { logConstructionInformation(); - if (hasBucketOwnershipTransfer()) { - markAllAvailableNodesAsRequiringRequest(); - } else { - updateSetOfNodesThatAreOutdated(); - addAdditionalNodesToOutdatedSet(outdatedNodes); - } - if (shouldRequestBucketInfo()) { - requestNodes(); - } + initializeBucketSpaceTransitions(false, outdatedNodesMap); } PendingClusterState::PendingClusterState( const framework::Clock& clock, const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, + DistributorBucketSpaceRepo &bucketSpaceRepo, api::Timestamp creationTimestamp) : _requestedNodes(clusterInfo->getStorageNodeCount()), - _outdatedNodes(clusterInfo->getStorageNodeCount()), - _iter(0), _prevClusterState(clusterInfo->getClusterState()), _newClusterState(clusterInfo->getClusterState()), _clock(clock), _clusterInfo(clusterInfo), _creationTimestamp(creationTimestamp), _sender(sender), - _bucketOwnershipTransfer(true) + _bucketSpaceRepo(bucketSpaceRepo), + _bucketOwnershipTransfer(true), + _pendingTransitions() { logConstructionInformation(); - markAllAvailableNodesAsRequiringRequest(); + initializeBucketSpaceTransitions(true, OutdatedNodesMap()); +} + +PendingClusterState::~PendingClusterState() {} + +void +PendingClusterState::initializeBucketSpaceTransitions(bool distributionChanged, const OutdatedNodesMap &outdatedNodesMap) +{ + OutdatedNodes emptyOutdatedNodes; + for (auto &elem : _bucketSpaceRepo) { + auto onItr = outdatedNodesMap.find(elem.first); + const auto &outdatedNodes = (onItr == outdatedNodesMap.end()) ? emptyOutdatedNodes : onItr->second; + auto pendingTransition = + std::make_unique<PendingBucketSpaceDbTransition> + (*this, *elem.second, distributionChanged, outdatedNodes, + _clusterInfo, _newClusterState, _creationTimestamp); + if (pendingTransition->getBucketOwnershipTransfer()) { + _bucketOwnershipTransfer = true; + } + _pendingTransitions.emplace(elem.first, std::move(pendingTransition)); + } if (shouldRequestBucketInfo()) { requestNodes(); } } -PendingClusterState::~PendingClusterState() {} - void PendingClusterState::logConstructionInformation() const { + const auto &distributorBucketSpace(_bucketSpaceRepo.get(BucketSpace::placeHolder())); + const auto &distribution(distributorBucketSpace.getDistribution()); LOG(debug, "New PendingClusterState constructed with previous cluster " "state '%s', new cluster state '%s', distribution config " "hash: '%s'", _prevClusterState.toString().c_str(), _newClusterState.toString().c_str(), - _clusterInfo->getDistribution().getNodeGraph().getDistributionConfigHash().c_str()); + distribution.getNodeGraph().getDistributionConfigHash().c_str()); } bool @@ -93,33 +110,14 @@ PendingClusterState::storageNodeUpInNewState(uint16_t node) const .getState().oneOf(_clusterInfo->getStorageUpStates()); } -void -PendingClusterState::markAllAvailableNodesAsRequiringRequest() +PendingClusterState::OutdatedNodesMap +PendingClusterState::getOutdatedNodesMap() const { - const uint16_t nodeCount(newStateStorageNodeCount()); - for (uint16_t i = 0; i < nodeCount; ++i) { - if (storageNodeUpInNewState(i)) { - _outdatedNodes.insert(i); - } + OutdatedNodesMap outdatedNodesMap; + for (const auto &elem : _pendingTransitions) { + outdatedNodesMap.emplace(elem.first, elem.second->getOutdatedNodes()); } -} - -void -PendingClusterState::addAdditionalNodesToOutdatedSet( - const std::unordered_set<uint16_t>& nodes) -{ - const uint16_t nodeCount(newStateStorageNodeCount()); - for (uint16_t node : nodes) { - if (node < nodeCount) { - _outdatedNodes.insert(node); - } - } -} - -std::unordered_set<uint16_t> -PendingClusterState::getOutdatedNodeSet() const -{ - return _outdatedNodes; + return outdatedNodesMap; } uint16_t @@ -157,47 +155,6 @@ PendingClusterState::iAmDown() const return myState.getState() == lib::State::DOWN; } -bool -PendingClusterState::storageNodeMayHaveLostData(uint16_t index) -{ - Node node(NodeType::STORAGE, index); - NodeState newState = _newClusterState.getNodeState(node); - NodeState oldState = _prevClusterState.getNodeState(node); - - return (newState.getStartTimestamp() > oldState.getStartTimestamp()); -} - -void -PendingClusterState::updateSetOfNodesThatAreOutdated() -{ - const uint16_t nodeCount(newStateStorageNodeCount()); - for (uint16_t index = 0; index < nodeCount; ++index) { - if (storageNodeMayHaveLostData(index) || storageNodeChanged(index)) { - _outdatedNodes.insert(index); - } - } -} - -bool -PendingClusterState::storageNodeChanged(uint16_t index) { - Node node(NodeType::STORAGE, index); - NodeState newState = _newClusterState.getNodeState(node); - NodeState oldNodeState = _prevClusterState.getNodeState(node); - - // similarTo() also covers disk states. - if (!(oldNodeState.similarTo(newState))) { - LOG(debug, - "State for storage node %d has changed from '%s' to '%s', " - "updating bucket information", - index, - oldNodeState.toString().c_str(), - newState.toString().c_str()); - return true; - } - - return false; -} - void PendingClusterState::requestNodes() { @@ -212,114 +169,33 @@ PendingClusterState::requestNodes() void PendingClusterState::requestBucketInfoFromStorageNodesWithChangedState() { - for (uint16_t idx : _outdatedNodes) { - if (storageNodeUpInNewState(idx)) { - requestNode(idx); - } - } -} - -bool -PendingClusterState::distributorChanged( - const lib::ClusterState& oldState, - const lib::ClusterState& newState) -{ - if (newState.getDistributionBitCount() != - oldState.getDistributionBitCount()) - { - return true; - } - - Node myNode(NodeType::DISTRIBUTOR, _sender.getDistributorIndex()); - if (oldState.getNodeState(myNode).getState() == - lib::State::DOWN) - { - return true; - } - - uint16_t oldCount = oldState.getNodeCount(NodeType::DISTRIBUTOR); - uint16_t newCount = newState.getNodeCount(NodeType::DISTRIBUTOR); - - uint16_t maxCount = std::max(oldCount, newCount); - - for (uint16_t i = 0; i < maxCount; ++i) { - Node node(NodeType::DISTRIBUTOR, i); - - const lib::State& old(oldState.getNodeState(node).getState()); - const lib::State& nw(newState.getNodeState(node).getState()); - - if (nodeWasUpButNowIsDown(old, nw)) { - return (nodeInSameGroupAsSelf(i) - || nodeNeedsOwnershipTransferFromGroupDown(i, newState)); + for (auto &elem : _pendingTransitions) { + const OutdatedNodes &outdatedNodes(elem.second->getOutdatedNodes()); + for (uint16_t idx : outdatedNodes) { + if (storageNodeUpInNewState(idx)) { + requestNode(BucketSpaceAndNode(elem.first, idx)); + } } } - - return false; -} - -bool -PendingClusterState::nodeWasUpButNowIsDown(const lib::State& old, - const lib::State& nw) const -{ - return (old.oneOf("uimr") && !nw.oneOf("uimr")); -} - -bool -PendingClusterState::nodeInSameGroupAsSelf(uint16_t index) const -{ - if (_clusterInfo->nodeInSameGroupAsSelf(index)) { - LOG(debug, - "Distributor %d state changed, need to request data from all " - "storage nodes", - index); - return true; - } else { - LOG(debug, - "Distributor %d state changed but unrelated to my group.", - index); - return false; - } -} - -bool -PendingClusterState::nodeNeedsOwnershipTransferFromGroupDown( - uint16_t nodeIndex, - const lib::ClusterState& state) const -{ - const lib::Distribution& dist(_clusterInfo->getDistribution()); - if (!dist.distributorAutoOwnershipTransferOnWholeGroupDown()) { - return false; // Not doing anything for downed groups. - } - const lib::Group* group(dist.getNodeGraph().getGroupForNode(nodeIndex)); - // If there is no group information associated with the node (because the - // group has changed or the node has been removed from config), we must - // also invoke ownership transfer of buckets. - if (group == nullptr - || lib::Distribution::allDistributorsDown(*group, state)) - { - LOG(debug, - "Distributor %u state changed and is in a " - "group that now has no distributors remaining", - nodeIndex); - return true; - } - return false; } void -PendingClusterState::requestNode(uint16_t node) +PendingClusterState::requestNode(BucketSpaceAndNode bucketSpaceAndNode) { - vespalib::string distributionHash(_clusterInfo->getDistributionHash()); + const auto &distributorBucketSpace(_bucketSpaceRepo.get(bucketSpaceAndNode.bucketSpace)); + const auto &distribution(distributorBucketSpace.getDistribution()); + vespalib::string distributionHash(distribution.getNodeGraph().getDistributionConfigHash()); LOG(debug, - "Requesting bucket info for node %d with cluster state '%s' " + "Requesting bucket info for bucket space %" PRIu64 " node %d with cluster state '%s' " "and distribution hash '%s'", - node, + bucketSpaceAndNode.bucketSpace.getId(), + bucketSpaceAndNode.node, _newClusterState.toString().c_str(), distributionHash.c_str()); std::shared_ptr<api::RequestBucketInfoCommand> cmd( new api::RequestBucketInfoCommand( - BucketSpace::placeHolder(), + bucketSpaceAndNode.bucketSpace, _sender.getDistributorIndex(), _newClusterState, distributionHash)); @@ -327,9 +203,9 @@ PendingClusterState::requestNode(uint16_t node) cmd->setPriority(api::StorageMessage::HIGH); cmd->setTimeout(INT_MAX); - _sentMessages[cmd->getMsgId()] = node; + _sentMessages.emplace(cmd->getMsgId(), bucketSpaceAndNode); - _sender.sendToNode(NodeType::STORAGE, node, cmd); + _sender.sendToNode(NodeType::STORAGE, bucketSpaceAndNode.node, cmd); } @@ -353,25 +229,20 @@ PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::Request if (iter == _sentMessages.end()) { return false; } - const uint16_t node = iter->second; + const BucketSpaceAndNode bucketSpaceAndNode = iter->second; if (!reply->getResult().success()) { framework::MilliSecTime resendTime(_clock); resendTime += framework::MilliSecTime(100); - _delayedRequests.push_back(std::make_pair(resendTime, node)); + _delayedRequests.emplace_back(resendTime, bucketSpaceAndNode); _sentMessages.erase(iter); return true; } - setNodeReplied(node); - - for (uint32_t i = 0; i < reply->getBucketInfo().size(); ++i) { - addNodeInfo(reply->getBucketInfo()[i]._bucketId, - BucketCopy(_creationTimestamp, - node, - reply->getBucketInfo()[i]._info)); - } - + setNodeReplied(bucketSpaceAndNode.node); + auto transitionIter = _pendingTransitions.find(bucketSpaceAndNode.bucketSpace); + assert(transitionIter != _pendingTransitions.end()); + transitionIter->second->onRequestBucketInfoReply(*reply, bucketSpaceAndNode.node); _sentMessages.erase(iter); return true; @@ -389,68 +260,8 @@ PendingClusterState::resendDelayedMessages() { } } -void -PendingClusterState::addNodeInfo( - const document::BucketId& id, - const BucketCopy& copy) -{ - _entries.push_back(Entry(id, copy)); -} - -PendingClusterState::Range -PendingClusterState::skipAllForSameBucket() -{ - Range r(_iter, _iter); - - for (document::BucketId& bid = _entries[_iter].bucketId; - _iter < _entries.size() && _entries[_iter].bucketId == bid; - ++_iter) - { - } - - r.second = _iter; - return r; -} - -void -PendingClusterState::insertInfo( - BucketDatabase::Entry& info, - const Range& range) -{ - std::vector<BucketCopy> copiesToAddOrUpdate( - getCopiesThatAreNewOrAltered(info, range)); - - std::vector<uint16_t> order( - _clusterInfo->getIdealStorageNodesForState( - _newClusterState, - _entries[range.first].bucketId)); - info->addNodes(copiesToAddOrUpdate, order, TrustedUpdate::DEFER); - - LOG_BUCKET_OPERATION_NO_LOCK( - _entries[range.first].bucketId, - vespalib::make_string("insertInfo: %s", - info.toString().c_str())); -} - -std::vector<BucketCopy> -PendingClusterState::getCopiesThatAreNewOrAltered( - BucketDatabase::Entry& info, - const Range& range) -{ - std::vector<BucketCopy> copiesToAdd; - for (uint32_t i = range.first; i < range.second; ++i) { - const BucketCopy& candidate(_entries[i].copy); - const BucketCopy* cp = info->getNode(candidate.getNode()); - - if (!cp || !(cp->getBucketInfo() == candidate.getBucketInfo())) { - copiesToAdd.push_back(candidate); - } - } - return copiesToAdd; -} - std::string -PendingClusterState::requestNodesToString() +PendingClusterState::requestNodesToString() const { std::ostringstream ost; for (uint32_t i = 0; i < _requestedNodes.size(); ++i) { @@ -464,136 +275,11 @@ PendingClusterState::requestNodesToString() return ost.str(); } -bool -PendingClusterState::removeCopiesFromNodesThatWereRequested( - BucketDatabase::Entry& e, - const document::BucketId& bucketId) -{ - bool updated = false; - for (uint32_t i = 0; i < e->getNodeCount();) { - auto& info(e->getNodeRef(i)); - const uint16_t entryNode(info.getNode()); - // Don't remove an entry if it's been updated in the time after the - // bucket info requests were sent, as this would erase newer state. - // Don't immediately update trusted state, as that could erroneously - // mark a single remaining replica as trusted even though there might - // be one or more additional replicas pending merge into the database. - if (nodeIsOutdated(entryNode) - && (info.getTimestamp() < _creationTimestamp) - && e->removeNode(entryNode, TrustedUpdate::DEFER)) - { - LOG(spam, - "Removed bucket %s from node %d", - bucketId.toString().c_str(), - entryNode); - updated = true; - // After removing current node, getNodeRef(i) will point to the _next_ node, so don't increment `i`. - } else { - ++i; - } - } - return updated; -} - -bool -PendingClusterState::databaseIteratorHasPassedBucketInfoIterator( - const document::BucketId& bucketId) const -{ - return (_iter < _entries.size() - && _entries[_iter].bucketId.toKey() < bucketId.toKey()); -} - -bool -PendingClusterState::bucketInfoIteratorPointsToBucket( - const document::BucketId& bucketId) const -{ - return _iter < _entries.size() && _entries[_iter].bucketId == bucketId; -} - -bool -PendingClusterState::process(BucketDatabase::Entry& e) -{ - document::BucketId bucketId(e.getBucketId()); - - LOG(spam, - "Before merging info from nodes [%s], bucket %s had info %s", - requestNodesToString().c_str(), - bucketId.toString().c_str(), - e.getBucketInfo().toString().c_str()); - - while (databaseIteratorHasPassedBucketInfoIterator(bucketId)) { - LOG(spam, "Found new bucket %s, adding", - _entries[_iter].bucketId.toString().c_str()); - - _missingEntries.push_back(skipAllForSameBucket()); - } - - bool updated(removeCopiesFromNodesThatWereRequested(e, bucketId)); - - if (bucketInfoIteratorPointsToBucket(bucketId)) { - LOG(spam, "Updating bucket %s", - _entries[_iter].bucketId.toString().c_str()); - - insertInfo(e, skipAllForSameBucket()); - updated = true; - } - - if (updated) { - // Remove bucket if we've previously removed all nodes from it - if (e->getNodeCount() == 0) { - _removedBuckets.push_back(bucketId); - } else { - e.getBucketInfo().updateTrusted(); - } - } - - LOG(spam, - "After merging info from nodes [%s], bucket %s had info %s", - requestNodesToString().c_str(), - bucketId.toString().c_str(), - e.getBucketInfo().toString().c_str()); - - return true; -} - void -PendingClusterState::addToBucketDB(BucketDatabase& db, - const Range& range) +PendingClusterState::mergeIntoBucketDatabases() { - LOG(spam, "Adding new bucket %s with %d copies", - _entries[range.first].bucketId.toString().c_str(), - range.second - range.first); - - BucketDatabase::Entry e(_entries[range.first].bucketId, BucketInfo()); - insertInfo(e, range); - if (e->getLastGarbageCollectionTime() == 0) { - e->setLastGarbageCollectionTime( - framework::MicroSecTime(_creationTimestamp) - .getSeconds().getTime()); - } - e.getBucketInfo().updateTrusted(); - db.update(e); -} - -void -PendingClusterState::mergeInto(BucketDatabase& db) -{ - std::sort(_entries.begin(), _entries.end()); - - db.forEach(*this); - - for (uint32_t i = 0; i < _removedBuckets.size(); ++i) { - db.remove(_removedBuckets[i]); - } - _removedBuckets.clear(); - - // All of the remaining were not already in the bucket database. - while (_iter < _entries.size()) { - _missingEntries.push_back(skipAllForSameBucket()); - } - - for (uint32_t i = 0; i < _missingEntries.size(); ++i) { - addToBucketDB(db, _missingEntries[i]); + for (auto &elem : _pendingTransitions) { + elem.second->mergeIntoBucketDatabase(); } } @@ -603,11 +289,9 @@ PendingClusterState::printXml(vespalib::XmlOutputStream& xos) const using namespace vespalib::xml; xos << XmlTag("systemstate_pending") << XmlAttribute("state", _newClusterState); - for (std::map<uint64_t, uint16_t>::const_iterator iter - = _sentMessages.begin(); iter != _sentMessages.end(); ++iter) - { + for (auto &elem : _sentMessages) { xos << XmlTag("pending") - << XmlAttribute("node", iter->second) + << XmlAttribute("node", elem.second.node) << XmlEndTag(); } xos << XmlEndTag(); @@ -621,4 +305,12 @@ PendingClusterState::getSummary() const (_clock.getTimeInMicros().getTime() - _creationTimestamp)); } +PendingBucketSpaceDbTransition & +PendingClusterState::getPendingBucketSpaceDbTransition(document::BucketSpace bucketSpace) +{ + auto transitionIter = _pendingTransitions.find(bucketSpace); + assert(transitionIter != _pendingTransitions.end()); + return *transitionIter->second; +} + } diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.h b/storage/src/vespa/storage/distributor/pendingclusterstate.h index 316d7996d81..2d75c795745 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.h +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.h @@ -1,43 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "pending_bucket_space_db_transition_entry.h" #include "clusterinformation.h" -#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/clock/clock.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/xmlserializable.h> -#include <unordered_set> +#include "outdated_nodes_map.h" +#include <unordered_map> #include <deque> namespace storage::distributor { class DistributorMessageSender; +class PendingBucketSpaceDbTransition; +class DistributorBucketSpaceRepo; /** * Class used by BucketDBUpdater to track request bucket info * messages sent to the storage nodes. */ -class PendingClusterState : public vespalib::XmlSerializable, - public BucketDatabase::MutableEntryProcessor { +class PendingClusterState : public vespalib::XmlSerializable { public: - struct Entry { - Entry(const document::BucketId& bid, - const BucketCopy& copy_) - : bucketId(bid), - copy(copy_) - {} - - document::BucketId bucketId; - BucketCopy copy; - - bool operator<(const Entry& other) const { - return bucketId.toKey() < other.bucketId.toKey(); - } - }; - + using OutdatedNodes = dbtransition::OutdatedNodes; + using OutdatedNodesMap = dbtransition::OutdatedNodesMap; struct Summary { Summary(const std::string& prevClusterState, const std::string& newClusterState, uint32_t processingTime); Summary(const Summary &); @@ -51,19 +40,18 @@ public: uint32_t _processingTime; }; - typedef std::vector<Entry> EntryList; - static std::unique_ptr<PendingClusterState> createForClusterStateChange( const framework::Clock& clock, const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, + DistributorBucketSpaceRepo &bucketSpaceRepo, const std::shared_ptr<api::SetSystemStateCommand>& newStateCmd, - const std::unordered_set<uint16_t>& outdatedNodes, + const OutdatedNodesMap &outdatedNodesMap, api::Timestamp creationTimestamp) { return std::unique_ptr<PendingClusterState>( - new PendingClusterState(clock, clusterInfo, sender, newStateCmd, - outdatedNodes, + new PendingClusterState(clock, clusterInfo, sender, bucketSpaceRepo, newStateCmd, + outdatedNodesMap, creationTimestamp)); } @@ -75,10 +63,11 @@ public: const framework::Clock& clock, const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, + DistributorBucketSpaceRepo &bucketSpaceRepo, api::Timestamp creationTimestamp) { return std::unique_ptr<PendingClusterState>( - new PendingClusterState(clock, clusterInfo, sender, creationTimestamp)); + new PendingClusterState(clock, clusterInfo, sender, bucketSpaceRepo, creationTimestamp)); } PendingClusterState(const PendingClusterState &) = delete; @@ -92,18 +81,13 @@ public: bool onRequestBucketInfoReply(const std::shared_ptr<api::RequestBucketInfoReply>& reply); /** - * Tags the given node as having replied to the - * request bucket info command. + * Tags the given node as having replied to at least one of the + * request bucket info commands. Only used for debug logging. */ void setNodeReplied(uint16_t nodeIdx) { _requestedNodes[nodeIdx] = true; } - /** - * Adds info from a node to our list of information. - */ - void addNodeInfo(const document::BucketId& id, const BucketCopy& copy); - /** Called to resend delayed resends due to failures. */ void resendDelayedMessages(); @@ -129,9 +113,6 @@ public: const lib::ClusterState& getPrevClusterState() const { return _prevClusterState; } - const lib::Distribution& getDistribution() const { - return _clusterInfo->getDistribution(); - } /** * Returns the union set of the outdated node set provided at construction @@ -140,22 +121,18 @@ public: * state was constructed for a distribution config change, this set will * be equal to the set of all available storage nodes. */ - std::unordered_set<uint16_t> getOutdatedNodeSet() const; + OutdatedNodesMap getOutdatedNodesMap() const; /** - * Merges all the results with the given bucket database. + * Merges all the results with the corresponding bucket databases. */ - void mergeInto(BucketDatabase& db); - bool process(BucketDatabase::Entry& e) override; - const EntryList& results() const { return _entries; } + void mergeIntoBucketDatabases(); + // Get pending transition for a specific bucket space. Only used by unit test. + PendingBucketSpaceDbTransition &getPendingBucketSpaceDbTransition(document::BucketSpace bucketSpace); - /** - * Returns true if this pending state was due to a distribution bit - * change rather than an actual state change. - */ - bool distributionChange() const { return _distributionChange; } void printXml(vespalib::XmlOutputStream&) const override; Summary getSummary() const; + std::string requestNodesToString() const; private: /** @@ -166,8 +143,9 @@ private: const framework::Clock&, const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, + DistributorBucketSpaceRepo &bucketSpaceRepo, const std::shared_ptr<api::SetSystemStateCommand>& newStateCmd, - const std::unordered_set<uint16_t>& outdatedNodes, + const OutdatedNodesMap &outdatedNodesMap, api::Timestamp creationTimestamp); /** @@ -178,16 +156,23 @@ private: const framework::Clock&, const ClusterInformation::CSP& clusterInfo, DistributorMessageSender& sender, + DistributorBucketSpaceRepo &bucketSpaceRepo, api::Timestamp creationTimestamp); + struct BucketSpaceAndNode { + document::BucketSpace bucketSpace; + uint16_t node; + BucketSpaceAndNode(document::BucketSpace bucketSpace_, + uint16_t node_) + : bucketSpace(bucketSpace_), + node(node_) + { + } + }; + + void initializeBucketSpaceTransitions(bool distributionChanged, const OutdatedNodesMap &outdatedNodesMap); void logConstructionInformation() const; - void requestNode(uint16_t node); - bool distributorChanged(const lib::ClusterState& oldState, const lib::ClusterState& newState); - bool storageNodeMayHaveLostData(uint16_t index); - bool storageNodeChanged(uint16_t index); - void markAllAvailableNodesAsRequiringRequest(); - void addAdditionalNodesToOutdatedSet(const std::unordered_set<uint16_t>& nodes); - void updateSetOfNodesThatAreOutdated(); + void requestNode(BucketSpaceAndNode bucketSpaceAndNode); void requestNodes(); void requestBucketInfoFromStorageNodesWithChangedState(); @@ -198,58 +183,14 @@ private: bool shouldRequestBucketInfo() const; bool clusterIsDown() const; bool iAmDown() const; - bool nodeInSameGroupAsSelf(uint16_t index) const; - bool nodeNeedsOwnershipTransferFromGroupDown(uint16_t nodeIndex, const lib::ClusterState& state) const; - bool nodeWasUpButNowIsDown(const lib::State& old, const lib::State& nw) const; - - typedef std::pair<uint32_t, uint32_t> Range; - - /** - * Skips through all entries for the same bucket and returns - * the range in the entry list for which they were found. - * The range is [from, to> - */ - Range skipAllForSameBucket(); - - void insertInfo(BucketDatabase::Entry& info, const Range& range); - void addToBucketDB(BucketDatabase& db, const Range& range); - - std::vector<BucketCopy> getCopiesThatAreNewOrAltered(BucketDatabase::Entry& info, const Range& range); - - std::string requestNodesToString(); - - // Returns whether at least one replica was removed from the entry. - // Does NOT implicitly update trusted status on remaining replicas; caller must do - // this explicitly. - bool removeCopiesFromNodesThatWereRequested(BucketDatabase::Entry& e, const document::BucketId& bucketId); - - bool databaseIteratorHasPassedBucketInfoIterator(const document::BucketId& bucketId) const; - bool bucketInfoIteratorPointsToBucket(const document::BucketId& bucketId) const; - - bool nodeIsOutdated(uint16_t node) const { - return (_outdatedNodes.find(node) != _outdatedNodes.end()); - } bool storageNodeUpInNewState(uint16_t node) const; std::shared_ptr<api::SetSystemStateCommand> _cmd; - std::map<uint64_t, uint16_t> _sentMessages; + std::map<uint64_t, BucketSpaceAndNode> _sentMessages; std::vector<bool> _requestedNodes; - std::vector<document::BucketId> _removedBuckets; - std::deque<std::pair<framework::MilliSecTime, uint16_t> > _delayedRequests; - - // Set for all nodes that may have changed state since that previous - // active cluster state, or that were marked as outdated when the pending - // cluster state was constructed. - // May be a superset of _requestedNodes, as some nodes that are outdated - // may be down and thus cannot get a request. - std::unordered_set<uint16_t> _outdatedNodes; - - EntryList _entries; - uint32_t _iter; - - std::vector<Range> _missingEntries; + std::deque<std::pair<framework::MilliSecTime, BucketSpaceAndNode> > _delayedRequests; lib::ClusterState _prevClusterState; lib::ClusterState _newClusterState; @@ -259,9 +200,10 @@ private: api::Timestamp _creationTimestamp; DistributorMessageSender& _sender; + DistributorBucketSpaceRepo &_bucketSpaceRepo; - bool _distributionChange; bool _bucketOwnershipTransfer; + std::unordered_map<document::BucketSpace, std::unique_ptr<PendingBucketSpaceDbTransition>, document::BucketSpace::hash> _pendingTransitions; }; } diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp index 6d562510f23..f7e207d7b8c 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp @@ -77,7 +77,7 @@ pairAsRange(Pair pair) std::vector<uint64_t> PendingMessageTracker::clearMessagesForNode(uint16_t node) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); MessagesByNodeAndBucket& idx(boost::multi_index::get<1>(_messages)); auto range = pairAsRange(idx.equal_range(boost::make_tuple(node))); @@ -95,7 +95,7 @@ void PendingMessageTracker::insert( const std::shared_ptr<api::StorageMessage>& msg) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (msg->getAddress()) { _messages.insert( MessageEntry(currentTime(), @@ -118,7 +118,7 @@ PendingMessageTracker::insert( document::Bucket PendingMessageTracker::reply(const api::StorageReply& r) { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); document::Bucket bucket; LOG(debug, "Got reply: %s", r.toString().c_str()); @@ -171,7 +171,7 @@ PendingMessageTracker::updateOperationStats(OperationStats& opStats, NodeStatsSnapshot PendingMessageTracker::getLatencyStatistics() const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); NodeStatsSnapshot snapshot; // Conveniently, snapshot data structure is exactly the same as our own. snapshot.nodeToStats = _nodeIndexToStats; @@ -205,7 +205,7 @@ PendingMessageTracker::checkPendingMessages(uint16_t node, const document::Bucket &bucket, Checker& checker) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(node, bucket))); @@ -216,7 +216,7 @@ void PendingMessageTracker::checkPendingMessages(const document::Bucket &bucket, Checker& checker) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByBucketAndType& msgs(boost::multi_index::get<2>(_messages)); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(bucket))); @@ -228,7 +228,7 @@ PendingMessageTracker::hasPendingMessage(uint16_t node, const document::Bucket &bucket, uint32_t messageType) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); auto range = msgs.equal_range(boost::make_tuple(node, bucket, messageType)); @@ -247,7 +247,7 @@ PendingMessageTracker::getStatusStartPage(std::ostream& out) const void PendingMessageTracker::getStatusPerBucket(std::ostream& out) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); using BucketMap = std::map<document::Bucket, std::vector<vespalib::string>>; @@ -285,7 +285,7 @@ PendingMessageTracker::getStatusPerBucket(std::ostream& out) const void PendingMessageTracker::getStatusPerNode(std::ostream& out) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); int lastNode = -1; for (MessagesByNodeAndBucket::const_iterator iter = @@ -337,7 +337,7 @@ PendingMessageTracker::print(std::ostream& /*out*/, NodeStats PendingMessageTracker::getNodeStats(uint16_t node) const { - vespalib::LockGuard guard(_lock); + std::lock_guard<std::mutex> guard(_lock); auto nodeIter = _nodeIndexToStats.find(node); return (nodeIter != _nodeIndexToStats.end() ? nodeIter->second : NodeStats()); diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index 48999cc837b..e7bcf85a38d 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -10,7 +10,6 @@ #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/util/sync.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> @@ -22,6 +21,7 @@ #include <set> #include <unordered_map> #include <chrono> +#include <mutex> namespace storage { namespace distributor { @@ -220,7 +220,7 @@ private: // Since distributor is currently single-threaded, this will only // contend when status page is being accessed. It is, however, required // to be present for that exact purpose. - vespalib::Lock _lock; + mutable std::mutex _lock; /** * Increment latency and operation count stats for the node the message diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp index a1b43149963..1519a9183ba 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp @@ -4,6 +4,8 @@ #include <vespa/storage/common/vectorprinter.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/persistence.h> +#include "distributor_bucket_space_repo.h" +#include "distributor_bucket_space.h" #include <vespa/log/log.h> @@ -123,8 +125,9 @@ PersistenceMessageTrackerImpl::canSendReplyEarly() const LOG(spam, "Can't return early because we have already replied or failed"); return false; } - - const lib::Distribution& distribution = _manager.getDistribution(); + auto &bucketSpaceRepo(_manager.getBucketSpaceRepo()); + auto &bucketSpace(bucketSpaceRepo.get(_reply->getBucket().getBucketSpace())); + const lib::Distribution& distribution = bucketSpace.getDistribution(); if (distribution.getInitialRedundancy() == 0) { LOG(spam, "Not returning early because initial redundancy wasn't set"); @@ -163,12 +166,14 @@ PersistenceMessageTrackerImpl::checkCopiesDeleted() // Don't check the buckets that have been remapped here, as we will // create them. + const auto &bucketSpaceRepo(_manager.getBucketSpaceRepo()); for (BucketInfoMap::const_iterator iter = _bucketInfo.begin(); iter != _bucketInfo.end(); iter++) { - BucketDatabase::Entry dbentry = - _manager.getBucketDatabase().get(iter->first.getBucketId()); + const auto &bucketSpace(bucketSpaceRepo.get(iter->first.getBucketSpace())); + const auto &bucketDb(bucketSpace.getBucketDatabase()); + BucketDatabase::Entry dbentry = bucketDb.get(iter->first.getBucketId()); if (!dbentry.valid()) { continue; diff --git a/storage/src/vespa/storage/distributor/simpleclusterinformation.h b/storage/src/vespa/storage/distributor/simpleclusterinformation.h index e6e46890c3f..2946abf620c 100644 --- a/storage/src/vespa/storage/distributor/simpleclusterinformation.h +++ b/storage/src/vespa/storage/distributor/simpleclusterinformation.h @@ -11,11 +11,9 @@ class SimpleClusterInformation : public ClusterInformation { public: SimpleClusterInformation(uint16_t myIndex, - const lib::Distribution& distribution, const lib::ClusterState& clusterState, const char* storageUpStates) : _myIndex(myIndex), - _distribution(distribution.serialize()), _clusterState(clusterState), _storageUpStates(storageUpStates) {} @@ -24,10 +22,6 @@ public: return _myIndex; } - const lib::Distribution& getDistribution() const override { - return _distribution; - } - const lib::ClusterState& getClusterState() const override { return _clusterState; } @@ -38,7 +32,6 @@ public: private: uint16_t _myIndex; - lib::Distribution _distribution; lib::ClusterState _clusterState; const char* _storageUpStates; }; diff --git a/storage/src/vespa/storage/distributor/statechecker.cpp b/storage/src/vespa/storage/distributor/statechecker.cpp index 0107430bb96..f959e5a80fb 100644 --- a/storage/src/vespa/storage/distributor/statechecker.cpp +++ b/storage/src/vespa/storage/distributor/statechecker.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statechecker.h" #include "distributorcomponent.h" +#include "distributor_bucket_space.h" #include <vespa/log/log.h> LOG_SETUP(".distributor.statechecker"); @@ -59,22 +60,23 @@ StateChecker::Result::createStoredResult( } StateChecker::Context::Context(const DistributorComponent& c, + const DistributorBucketSpace &distributorBucketSpace, NodeMaintenanceStatsTracker& statsTracker, - const document::BucketId& bid) - : bucketId(bid), - siblingBucket(c.getSibling(bid)), + const document::Bucket &bucket_) + : bucket(bucket_), + siblingBucket(c.getSibling(bucket.getBucketId())), systemState(c.getClusterState()), distributorConfig(c.getDistributor().getConfig()), - distribution(c.getDistribution()), + distribution(distributorBucketSpace.getDistribution()), gcTimeCalculator(c.getDistributor().getBucketIdHasher(), std::chrono::seconds(distributorConfig .getGarbageCollectionInterval())), component(c), - db(c.getBucketDatabase()), + db(distributorBucketSpace.getBucketDatabase()), stats(statsTracker) { idealState = - distribution.getIdealStorageNodes(systemState, bucketId); + distribution.getIdealStorageNodes(systemState, bucket.getBucketId()); unorderedIdealState.insert(idealState.begin(), idealState.end()); } diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index fbadd5642d4..e204cf5325a 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -2,7 +2,7 @@ #pragma once #include "bucketgctimecalculator.h" -#include "maintenancebucket.h" +#include <vespa/storage/distributor/maintenance/maintenancepriority.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/bucketdb/bucketdatabase.h> @@ -19,6 +19,7 @@ class DistributorConfiguration; namespace distributor { class DistributorComponent; +class DistributorBucketSpace; class NodeMaintenanceStatsTracker; /** @@ -45,15 +46,16 @@ public: struct Context { Context(const DistributorComponent&, + const DistributorBucketSpace &distributorBucketSpace, NodeMaintenanceStatsTracker&, - const document::BucketId& bid); + const document::Bucket &bucket_); ~Context(); Context(const Context &) = delete; Context & operator =(const Context &) = delete; // Per bucket - document::BucketId bucketId; + document::Bucket bucket; document::BucketId siblingBucket; BucketDatabase::Entry entry; @@ -82,7 +84,8 @@ public: return siblingEntry; } - document::Bucket getBucket() const { return document::Bucket(document::BucketSpace::placeHolder(), bucketId); } + document::Bucket getBucket() const { return bucket; } + document::BucketId getBucketId() const { return bucket.getBucketId(); } std::string toString() const; }; diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index 35d111a8c38..1f0cb19ef93 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -27,12 +27,12 @@ SplitBucketStateChecker::validForSplit(StateChecker::Context& c) if (c.entry->getNodeCount() == 0) { LOG(spam, "Can't split bucket %s, since it has no copies", - c.bucketId.toString().c_str()); + c.bucket.toString().c_str()); return false; } // Can't split anymore if we already used 58 bits. - if (c.bucketId.getUsedBits() >= 58) { + if (c.getBucketId().getUsedBits() >= 58) { return false; } @@ -145,7 +145,7 @@ SplitBucketStateChecker::check(StateChecker::Context& c) { } // Always split it if it has less used bits than the minimum. - if (c.bucketId.getUsedBits() < c.distributorConfig.getMinimalBucketSplit()) { + if (c.getBucketId().getUsedBits() < c.distributorConfig.getMinimalBucketSplit()) { return generateMinimumBucketSplitOperation(c); } return Result::noMaintenanceNeeded(); @@ -217,7 +217,7 @@ JoinBucketsStateChecker::siblingsAreInSync(const Context& context) const LOG(spam, "Not joining bucket %s because sibling bucket %s had different " "node count", - context.bucketId.toString().c_str(), + context.bucket.toString().c_str(), context.siblingBucket.toString().c_str()); return false; } @@ -238,7 +238,7 @@ JoinBucketsStateChecker::siblingsAreInSync(const Context& context) const "does not have the same node set, or inconsistent joins cannot be " "performed either due to config or because replicas were not in " "their ideal location", - context.bucketId.toString().c_str(), + context.bucket.toString().c_str(), context.siblingBucket.toString().c_str()); return false; } @@ -247,7 +247,7 @@ JoinBucketsStateChecker::siblingsAreInSync(const Context& context) const LOG(spam, "Not joining bucket %s because it or %s is out of sync " "and syncing it may cause it to become too large", - context.bucketId.toString().c_str(), + context.bucket.toString().c_str(), context.siblingBucket.toString().c_str()); return false; } @@ -258,8 +258,8 @@ JoinBucketsStateChecker::siblingsAreInSync(const Context& context) const bool JoinBucketsStateChecker::singleBucketJoinIsConsistent(const Context& c) const { - document::BucketId joinTarget(c.bucketId.getUsedBits() - 1, - c.bucketId.getRawId()); + document::BucketId joinTarget(c.getBucketId().getUsedBits() - 1, + c.getBucketId().getRawId()); // If there are 2 children under the potential join target bucket, joining // would cause the bucket tree to become inconsistent. The reason for this // being that "moving" a bucket one bit up in the tree (and into @@ -305,30 +305,30 @@ JoinBucketsStateChecker::shouldJoin(const Context& c) const { if (c.entry->getNodeCount() == 0) { LOG(spam, "Not joining bucket %s because it has no nodes", - c.bucketId.toString().c_str()); + c.bucket.toString().c_str()); return false; } if (contextBucketHasTooManyReplicas(c)) { LOG(spam, "Not joining %s because it has too high replication level", - c.bucketId.toString().c_str()); + c.bucket.toString().c_str()); return false; } if (c.distributorConfig.getJoinSize() == 0 && c.distributorConfig.getJoinCount() == 0) { LOG(spam, "Not joining bucket %s because join is disabled", - c.bucketId.toString().c_str()); + c.bucket.toString().c_str()); return false; } - if (bucketAtDistributionBitLimit(c.bucketId, c)) { + if (bucketAtDistributionBitLimit(c.getBucketId(), c)) { LOG(spam, "Not joining bucket %s because it is below the min split " "count (config: %u, cluster state: %u, bucket has: %u)", - c.bucketId.toString().c_str(), + c.bucket.toString().c_str(), c.distributorConfig.getMinimalBucketSplit(), c.systemState.getDistributionBitCount(), - c.bucketId.getUsedBits()); + c.getBucketId().getUsedBits()); return false; } @@ -337,11 +337,11 @@ JoinBucketsStateChecker::shouldJoin(const Context& c) const } if (c.getSiblingEntry().valid()) { - if (!isFirstSibling(c.bucketId)) { + if (!isFirstSibling(c.getBucketId())) { LOG(spam, "Not joining bucket %s because it is the second sibling of " "%s and not the first", - c.bucketId.toString().c_str(), + c.bucket.toString().c_str(), c.siblingBucket.toString().c_str()); return false; } @@ -427,8 +427,8 @@ JoinBucketsStateChecker::computeJoinBucket(const Context& c) const { // Always decrease by at least 1 bit, as we could not get here unless this // were a valid outcome. - unsigned int level = c.bucketId.getUsedBits() - 1; - document::BucketId target(level, c.bucketId.getRawId()); + unsigned int level = c.getBucketId().getUsedBits() - 1; + document::BucketId target(level, c.getBucketId().getRawId()); // Push bucket up the tree as long as it gets no siblings. This means // joins involving 2 source buckets will currently only be decreased by 1 @@ -436,7 +436,7 @@ JoinBucketsStateChecker::computeJoinBucket(const Context& c) const // be decreased by multiple bits. We may want to optimize joins for cases // with 2 source buckets in the future. while (true) { - document::BucketId candidate(level, c.bucketId.getRawId()); + document::BucketId candidate(level, c.getBucketId().getRawId()); if (bucketHasMultipleChildren(candidate, c) || !legalBucketSplitLevel(candidate, c)) { @@ -445,7 +445,7 @@ JoinBucketsStateChecker::computeJoinBucket(const Context& c) const --level; target = candidate; } - return document::Bucket(BucketSpace::placeHolder(), target); + return document::Bucket(c.getBucket().getBucketSpace(), target); } StateChecker::Result @@ -458,15 +458,15 @@ JoinBucketsStateChecker::check(StateChecker::Context& c) } document::Bucket joinedBucket(computeJoinBucket(c)); - assert(joinedBucket.getBucketId().getUsedBits() < c.bucketId.getUsedBits()); + assert(joinedBucket.getBucketId().getUsedBits() < c.getBucketId().getUsedBits()); std::vector<document::BucketId> sourceBuckets; if (c.getSiblingEntry().valid()) { sourceBuckets.push_back(c.siblingBucket); } else { - sourceBuckets.push_back(c.bucketId); + sourceBuckets.push_back(c.getBucketId()); } - sourceBuckets.push_back(c.bucketId); + sourceBuckets.push_back(c.getBucketId()); IdealStateOperation::UP op(new JoinOperation( c.component.getClusterName(), BucketAndNodes(joinedBucket, c.entry->getNodes()), @@ -568,7 +568,7 @@ SplitInconsistentStateChecker::check(StateChecker::Context& c) return Result::noMaintenanceNeeded(); } - if (!isLeastSplitBucket(c.bucketId, c.entries)) { + if (!isLeastSplitBucket(c.getBucketId(), c.entries)) { return Result::noMaintenanceNeeded(); } @@ -581,7 +581,7 @@ SplitInconsistentStateChecker::check(StateChecker::Context& c) op->setPriority(c.distributorConfig.getMaintenancePriorities() .splitInconsistentBucket); - op->setDetailedReason(getReason(c.bucketId, c.entries)); + op->setDetailedReason(getReason(c.getBucketId(), c.entries)); return Result::createStoredResult(std::move(op), MaintenancePriority::HIGH); } @@ -849,14 +849,14 @@ SynchronizeAndMoveStateChecker::check(StateChecker::Context& c) op->setPriority(result.priority()); op->setDetailedReason(result.reason()); MaintenancePriority::Priority schedPri( - result.needsMoveOnly() ? MaintenancePriority::VERY_LOW + result.needsMoveOnly() ? MaintenancePriority::LOW : MaintenancePriority::MEDIUM); return Result::createStoredResult(std::move(op), schedPri); } else { LOG(spam, "Bucket %s: No need for merge, as bucket is in consistent state " "(or inconsistent buckets are empty) %s", - c.bucketId.toString().c_str(), + c.bucket.toString().c_str(), c.entry->toString().c_str()); return Result::noMaintenanceNeeded(); } @@ -1119,7 +1119,7 @@ GarbageCollectionStateChecker::needsGarbageCollection(const Context& c) const std::chrono::seconds currentTime( c.component.getClock().getTimeInSeconds().getTime()); - return c.gcTimeCalculator.shouldGc(c.bucketId, currentTime, lastRunAt); + return c.gcTimeCalculator.shouldGc(c.getBucketId(), currentTime, lastRunAt); } StateChecker::Result @@ -1142,7 +1142,7 @@ GarbageCollectionStateChecker::check(Context& c) op->setPriority(c.distributorConfig.getMaintenancePriorities() .garbageCollection); op->setDetailedReason(reason.c_str()); - return Result::createStoredResult(std::move(op), MaintenancePriority::LOW); + return Result::createStoredResult(std::move(op), MaintenancePriority::VERY_LOW); } else { return Result::noMaintenanceNeeded(); } diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index da734c07c2d..6fd75e06d9d 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -36,4 +36,14 @@ ServiceLayerComponentRegisterImpl::setDiskCount(uint16_t count) } } +void +ServiceLayerComponentRegisterImpl::setDistribution(lib::Distribution::SP distribution) +{ + // For now, copy distribution to all content bucket spaces + for (const auto &elem : _bucketSpaceRepo) { + elem.second->setDistribution(distribution); + } + StorageComponentRegisterImpl::setDistribution(distribution); +} + } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index 5b3e54e3831..df4047c92c3 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -37,6 +37,7 @@ public: void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; void setDiskCount(uint16_t count); + virtual void setDistribution(lib::Distribution::SP distribution) override; }; } // storage diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index ae028dd20bd..900fa71d7a0 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -3,6 +3,7 @@ #include "bucketownershipnotifier.h" #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/bucketoperationlogger.h> +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/backtrace.h> @@ -19,7 +20,8 @@ BucketOwnershipNotifier::getOwnerDistributorForBucket( const document::Bucket &bucket) const { try { - return (_component.getDistribution()->getIdealDistributorNode( + auto distribution(_component.getBucketSpaceRepo().get(bucket.getBucketSpace()).getDistribution()); + return (distribution->getIdealDistributorNode( *_component.getStateUpdater().getSystemState(), bucket.getBucketId())); // If we get exceptions there aren't any distributors, so they'll have // to explicitly fetch all bucket info eventually anyway. diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index fffd559866d..31f712faea2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -742,7 +742,7 @@ FileStorHandlerImpl::calculateTargetBasedOnDocId( std::vector<RemapInfo*>& targets) { document::DocumentId id(getDocId(msg)); - document::Bucket bucket(BucketSpace::placeHolder(), _bucketIdFactory.getBucketId(id)); + document::Bucket bucket(msg.getBucket().getBucketSpace(), _bucketIdFactory.getBucketId(id)); for (uint32_t i = 0; i < targets.size(); i++) { if (targets[i]->bucket.getBucketId().getRawId() != 0 && diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 88a7343f8c8..cf41a297541 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -966,7 +966,6 @@ void FileStorManager::updateState() { lib::ClusterState::CSP state(_component.getStateUpdater().getSystemState()); - spi::ClusterState spiState(*state, _component.getIndex(), *_component.getDistribution()); lib::Node node(_component.getNodeType(), _component.getIndex()); bool nodeUp = state->getNodeState(node).getState().oneOf("uir"); @@ -977,7 +976,11 @@ FileStorManager::updateState() Deactivator deactivator; _component.getBucketSpaceRepo().forEachBucket(deactivator, "FileStorManager::updateState"); } - _provider->setClusterState(spiState); + for (const auto &elem : _component.getBucketSpaceRepo()) { + BucketSpace bucketSpace(elem.first); + spi::ClusterState spiState(*state, _component.getIndex(), *elem.second->getDistribution()); + _provider->setClusterState(bucketSpace, spiState); + } _nodeUpInLastNodeStateSeenByProvider = nodeUp; } diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp index 093576622db..1834c93209d 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp @@ -12,6 +12,32 @@ using document::BucketSpace; namespace storage { +ModifiedBucketChecker::CyclicBucketSpaceIterator:: +CyclicBucketSpaceIterator(const ContentBucketSpaceRepo::BucketSpaces &bucketSpaces) + : _bucketSpaces(bucketSpaces), + _idx(0) +{ + std::sort(_bucketSpaces.begin(), _bucketSpaces.end()); +} + +ModifiedBucketChecker::BucketIdListResult::BucketIdListResult() + : _bucketSpace(document::BucketSpace::placeHolder()), + _buckets() +{ +} + +void +ModifiedBucketChecker::BucketIdListResult::reset(document::BucketSpace bucketSpace, + document::bucket::BucketIdList &buckets) +{ + _bucketSpace = bucketSpace; + assert(_buckets.empty()); + _buckets.swap(buckets); + // We pick chunks from the end of the list, so reverse it to get + // the same send order as order received. + std::reverse(_buckets.begin(), _buckets.end()); +} + ModifiedBucketChecker::ModifiedBucketChecker( ServiceLayerComponentRegister& compReg, spi::PersistenceProvider& provider, @@ -23,6 +49,8 @@ ModifiedBucketChecker::ModifiedBucketChecker( _configFetcher(configUri.getContext()), _monitor(), _stateLock(), + _bucketSpaces(), + _rechecksNotStarted(), _pendingRequests(0), _maxPendingChunkSize(100), _singleThreadMode(false) @@ -33,6 +61,7 @@ ModifiedBucketChecker::ModifiedBucketChecker( std::ostringstream threadName; threadName << "Modified bucket checker " << static_cast<void*>(this); _component.reset(new ServiceLayerComponent(compReg, threadName.str())); + _bucketSpaces = std::make_unique<CyclicBucketSpaceIterator>(_component->getBucketSpaceRepo().getBucketSpaces()); } ModifiedBucketChecker::~ModifiedBucketChecker() @@ -120,9 +149,9 @@ ModifiedBucketChecker::onInternalReply( } bool -ModifiedBucketChecker::requestModifiedBucketsFromProvider() +ModifiedBucketChecker::requestModifiedBucketsFromProvider(document::BucketSpace bucketSpace) { - spi::BucketIdListResult result(_provider.getModifiedBuckets(document::BucketSpace::placeHolder())); + spi::BucketIdListResult result(_provider.getModifiedBuckets(bucketSpace)); if (result.hasError()) { LOG(debug, "getModifiedBuckets() failed: %s", result.toString().c_str()); @@ -130,11 +159,7 @@ ModifiedBucketChecker::requestModifiedBucketsFromProvider() } { vespalib::LockGuard guard(_stateLock); - assert(_rechecksNotStarted.empty()); - _rechecksNotStarted.swap(result.getList()); - // We pick chunks from the end of the list, so reverse it to get - // the same send order as order received. - std::reverse(_rechecksNotStarted.begin(), _rechecksNotStarted.end()); + _rechecksNotStarted.reset(bucketSpace, result.getList()); } return true; } @@ -148,7 +173,7 @@ ModifiedBucketChecker::nextRecheckChunk( size_t n = std::min(_maxPendingChunkSize, _rechecksNotStarted.size()); for (size_t i = 0; i < n; ++i) { - document::Bucket bucket(BucketSpace::placeHolder(), _rechecksNotStarted.back()); + document::Bucket bucket(_rechecksNotStarted.bucketSpace(), _rechecksNotStarted.back()); commandsToSend.emplace_back(new RecheckBucketInfoCommand(bucket)); _rechecksNotStarted.pop_back(); } @@ -184,7 +209,7 @@ ModifiedBucketChecker::tick() shouldRequestFromProvider = !moreChunksRemaining(); } if (shouldRequestFromProvider) { - if (!requestModifiedBucketsFromProvider()) { + if (!requestModifiedBucketsFromProvider(_bucketSpaces->next())) { return false; } } diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h index 3e43481bf49..c6f13ce1a4c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelink.h> @@ -49,17 +50,45 @@ private: bool moreChunksRemaining() const { return !_rechecksNotStarted.empty(); } - bool requestModifiedBucketsFromProvider(); + bool requestModifiedBucketsFromProvider(document::BucketSpace bucketSpace); void nextRecheckChunk(std::vector<RecheckBucketInfoCommand::SP>&); void dispatchAllToPersistenceQueues(const std::vector<RecheckBucketInfoCommand::SP>&); + class CyclicBucketSpaceIterator { + private: + ContentBucketSpaceRepo::BucketSpaces _bucketSpaces; + size_t _idx; + public: + using UP = std::unique_ptr<CyclicBucketSpaceIterator>; + CyclicBucketSpaceIterator(const ContentBucketSpaceRepo::BucketSpaces &bucketSpaces); + document::BucketSpace next() { + return _bucketSpaces[(_idx++)%_bucketSpaces.size()]; + } + }; + + class BucketIdListResult { + private: + document::BucketSpace _bucketSpace; + document::bucket::BucketIdList _buckets; + public: + BucketIdListResult(); + void reset(document::BucketSpace bucketSpace, + document::bucket::BucketIdList &buckets); + const document::BucketSpace &bucketSpace() const { return _bucketSpace; } + size_t size() const { return _buckets.size(); } + bool empty() const { return _buckets.empty(); } + const document::BucketId &back() const { return _buckets.back(); } + void pop_back() { _buckets.pop_back(); } + }; + spi::PersistenceProvider& _provider; ServiceLayerComponent::UP _component; framework::Thread::UP _thread; config::ConfigFetcher _configFetcher; vespalib::Monitor _monitor; vespalib::Lock _stateLock; - document::bucket::BucketIdList _rechecksNotStarted; + CyclicBucketSpaceIterator::UP _bucketSpaces; + BucketIdListResult _rechecksNotStarted; size_t _pendingRequests; size_t _maxPendingChunkSize; bool _singleThreadMode; // For unit testing only diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 15b0a469b35..056561e8e21 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -55,9 +55,9 @@ ProviderErrorWrapper::listBuckets(BucketSpace bucketSpace, spi::PartitionId part } spi::Result -ProviderErrorWrapper::setClusterState(const spi::ClusterState& state) +ProviderErrorWrapper::setClusterState(BucketSpace bucketSpace, const spi::ClusterState& state) { - return checkResult(_impl.setClusterState(state)); + return checkResult(_impl.setClusterState(bucketSpace, state)); } spi::Result diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 122837e75ed..3b5ace90d13 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -44,7 +44,7 @@ public: spi::Result initialize() override; spi::PartitionStateListResult getPartitionStates() const override; spi::BucketIdListResult listBuckets(BucketSpace bucketSpace, spi::PartitionId) const override; - spi::Result setClusterState(const spi::ClusterState&) override; + spi::Result setClusterState(BucketSpace bucketSpace, const spi::ClusterState&) override; spi::Result setActiveState(const spi::Bucket& bucket, spi::BucketInfo::ActiveState newState) override; spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; spi::Result put(const spi::Bucket&, spi::Timestamp, const spi::DocumentSP&, spi::Context&) override; diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp index 5a3c7e5c35c..b4c7d1e3e80 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp @@ -19,31 +19,6 @@ using document::BucketSpace; namespace storage { namespace { - /* - std::string printDate(time_t time) { - char date[26]; - struct tm datestruct; - struct tm* datestructptr = gmtime_r(&time, &datestruct); - assert(datestructptr); - char* result = asctime_r(&datestruct, date); - size_t size = strlen(result); - while (size > 0) { - bool stop = false; - switch (result[size - 1]) { - case '\n': - case '\r': - case '\f': - case '\t': - --size; - default: - stop = true; - break; - } - if (stop) break; - } - return std::string(result, size); - } - */ std::string printMinutesOfDay(uint32_t minutesOfDay) { std::ostringstream ost; @@ -131,7 +106,7 @@ struct NextEntryFinder { _first = false; return StorBucketDatabase::CONTINUE; } else { - _next.reset(new document::BucketId(bucket)); + _next = std::make_unique<document::BucketId>(bucket); return StorBucketDatabase::ABORT; } } @@ -145,48 +120,58 @@ std::unique_ptr<document::BucketId> getNextId(StorBucketDatabase& database, database.each(proc, "BucketIntegrityChecker::getNextId", last.toKey()); return std::move(proc._next); } + +bool allBucketSpacesExhausted(size_t index, const ContentBucketSpaceRepo::BucketSpaces& bucketSpaces) noexcept { + return (index == bucketSpaces.size() - 1); +} + } // End of anonymous namespace -document::BucketId -BucketIntegrityChecker::DiskData::iterate(StorBucketDatabase& bucketDatabase) +document::Bucket +BucketIntegrityChecker::DiskData::iterate(const ContentBucketSpaceRepo::BucketSpaces& bucketSpaces, + const ContentBucketSpaceRepo& bucketSpaceRepo) { static uint32_t i=0; - // Resend failed buckets once in a while - if (failedRepairs.size() > 0 && ++i % 10 == 9) - { - document::BucketId bid(failedRepairs.front()); + // Resend failed buckets once in a while + if (!failedRepairs.empty() && ++i % 10 == 9) { + document::Bucket bucket(failedRepairs.front()); LOG(spam, "Scheduling next bucket %s from failed repairs list", - bid.toString().c_str()); + bucket.toString().c_str()); failedRepairs.pop_front(); ++retriedBuckets; - return bid; + return bucket; } if (state == NOT_STARTED) { - // Guarantueed to be before all buckets. + // Guaranteed to be before all buckets. currentBucket = document::BucketId(0, 0); + currentBucketSpaceIndex = 0; } - if (state != DONE) { - std::unique_ptr<document::BucketId> bid( - getNextId(bucketDatabase, currentBucket, disk)); - if (bid.get()) { + while (state != DONE) { + const auto currentSpace = bucketSpaces[currentBucketSpaceIndex]; + const auto bid = getNextId(bucketSpaceRepo.get(currentSpace).bucketDatabase(), currentBucket, disk); + if (bid) { state = IN_PROGRESS; currentBucket = *bid; - return currentBucket; - } else { + return document::Bucket(currentSpace, currentBucket); + } else if (allBucketSpacesExhausted(currentBucketSpaceIndex, bucketSpaces)) { state = DONE; + break; + } else { + ++currentBucketSpaceIndex; + currentBucket = document::BucketId(0, 0); } } - // If we didn't schedule repaired, but we ended up not having any other, - // take repaired once anyways - if (failedRepairs.size() > 0) { - document::BucketId bid(failedRepairs.front()); + // If we didn't schedule repaired, but we ended up not having any other, + // take repaired once anyways + if (!failedRepairs.empty()) { + document::Bucket bucket(failedRepairs.front()); LOG(spam, "Done iterating, scheduling next bucket %s from failed " - "repairs list", bid.toString().c_str()); + "repairs list", bucket.toString().c_str()); failedRepairs.pop_front(); ++retriedBuckets; - return bid; + return bucket; } - return document::BucketId(0, 0); + return document::Bucket(bucketSpaces[currentBucketSpaceIndex], document::BucketId(0, 0)); } BucketIntegrityChecker::BucketIntegrityChecker( @@ -196,7 +181,9 @@ BucketIntegrityChecker::BucketIntegrityChecker( Runnable(), framework::HtmlStatusReporter("bucketintegritychecker", "Bucket integrity checker"), + _component(compReg, "bucketintegritychecker"), _cycleCount(0), + _bucketSpaces(_component.getBucketSpaceRepo().getBucketSpaces()), _status(), _lastCycleStart(0), _cycleStartBucketCount(0), @@ -205,19 +192,18 @@ BucketIntegrityChecker::BucketIntegrityChecker( _currentRunWithFullVerification(false), _verifyAllRepairs(false), _scheduleOptions(), - _systemState(), _wait(), _configFetcher(configUri.getContext()), - _maxThreadWaitTime(60 * 1000), - _component(compReg, "bucketintegritychecker") + _maxThreadWaitTime(60 * 1000) { + assert(!_bucketSpaces.empty()); LOG(debug, "Configuring bucket integrity checker to work with %u disks.", _component.getDiskCount()); _status.resize(_component.getDiskCount()); for (uint16_t i=0; i<_component.getDiskCount(); ++i) { _status[i].disk = i; } - if (_status.size() == 0) { + if (_status.empty()) { throw vespalib::IllegalStateException( "Cannot have storage with no disks.", VESPA_STRLOC); } @@ -254,10 +240,10 @@ BucketIntegrityChecker::~BucketIntegrityChecker() void BucketIntegrityChecker::onClose() { - // Avoid getting config during shutdown + // Avoid getting config during shutdown _configFetcher.close(); - // Close thread to ensure we don't send anything more down after - if (_thread.get() != 0) { + // Close thread to ensure we don't send anything more down after + if (_thread) { LOG(debug, "Waiting for bucket integrity worker thread to close."); _thread->interruptAndJoin(&_wait); LOG(debug, "Bucket integrity worker thread closed."); @@ -367,18 +353,20 @@ bool BucketIntegrityChecker::onInternalReply( const std::shared_ptr<api::InternalReply>& internalReply) { - // We only care about repair bucket replies - shared_ptr<RepairBucketReply> reply( - std::dynamic_pointer_cast<RepairBucketReply>(internalReply)); - if (!reply.get()) return false; + // We only care about repair bucket replies + auto reply = std::dynamic_pointer_cast<RepairBucketReply>(internalReply); + if (!reply) { + return false; + } vespalib::MonitorGuard monitor(_wait); _lastResponseTime = _component.getClock().getTimeInSeconds(); uint8_t disk = reply->getDisk(); + assert(disk < _status.size()); --_status[disk].pendingCount; LOG(spam, "Got repair reply for bucket %s: %s. %u messages still pending " "for disk %u. Bucket altered ? %s", - reply->getBucketId().toString().c_str(), + reply->getBucket().toString().c_str(), reply->getResult().toString().c_str(), _status[disk].pendingCount, disk, (reply->bucketAltered() ? "true" : "false")); @@ -399,13 +387,13 @@ BucketIntegrityChecker::onInternalReply( ++_status[disk].checkedBuckets; LOGBP(debug, "Failed to repair bucket %s due to aborting request. " "Likely bucket split/join or storage shutting down: %s", - reply->getBucketId().toString().c_str(), + reply->getBucket().toString().c_str(), reply->getResult().toString().c_str()); } else { - _status[disk].failedRepairs.push_back(reply->getBucketId()); + _status[disk].failedRepairs.push_back(reply->getBucket()); LOGBP(warning, "Failed to perform maintenance on bucket %s, " "scheduled to be retried: %s", - reply->getBucketId().toString().c_str(), + reply->getBucket().toString().c_str(), reply->getResult().toString().c_str()); } if (_lastCycleCompleted) { @@ -415,16 +403,6 @@ BucketIntegrityChecker::onInternalReply( return true; } -bool -BucketIntegrityChecker::onSetSystemState( - const std::shared_ptr<api::SetSystemStateCommand>& cmd) -{ - vespalib::MonitorGuard monitor(_wait); - _systemState = cmd->getSystemState(); - return false; -} - - SchedulingOptions::RunState BucketIntegrityChecker::getCurrentRunState( framework::SecondTime currentTime) const @@ -449,9 +427,7 @@ BucketIntegrityChecker::getCurrentRunState( ) ) { // If we're within region in day that we can run. -//std::cerr << "We're inside time boundary. Current time: " << minutesOfDay << " (" << printMinutesOfDay(minutesOfDay) << "). Running between " << _scheduleOptions._dailyCycleStart << " (" << printMinutesOfDay(_scheduleOptions._dailyCycleStart) << ") - " << _scheduleOptions._dailyCycleStop << " (" << printMinutesOfDay(_scheduleOptions._dailyCycleStop) << ")\n"; if (state == SchedulingOptions::CONTINUE) { -//std::cerr << "We're in continue state.\n"; // If we're in a continue state, set runstate if there's a current // run active that isn't completed yet, don't run otherwise. state = (_lastCycleCompleted @@ -471,18 +447,13 @@ BucketIntegrityChecker::getCurrentRunState( if (_currentRunWithFullVerification || state == SchedulingOptions::RUN_CHEAP) { -//std::cerr << "Tagging dont run since too little time passed since last run\n" << "current time: " << currentTime << ", last start " << _lastCycleStart << ", min cycle time " << _scheduleOptions._minCycleTime << "\n"; state = SchedulingOptions::DONT_RUN; } else { -//std::cerr << "We can start new run. Last cycle started at " << _lastCycleStart.toString() << " current time is " << currentTime.toString() << " and min cycle time is " << _scheduleOptions._minCycleTime << "\n"; } - } else { -//std::cerr << "Enough time passed? " << currentTime.toString() << " - " << _lastCycleStart.toString() << " >= " << _scheduleOptions._minCycleTime << "\n"; } } } else { // If we're outside of time of day boundaries, don't run -//std::cerr << "We're outside time boundary. Current time: " << minutesOfDay << " (" << printMinutesOfDay(minutesOfDay) << "). Only running between " << _scheduleOptions._dailyCycleStart << " (" << printMinutesOfDay(_scheduleOptions._dailyCycleStart) << ") - " << _scheduleOptions._dailyCycleStop << " (" << printMinutesOfDay(_scheduleOptions._dailyCycleStop) << ")\n"; state = SchedulingOptions::DONT_RUN; } return state; @@ -493,7 +464,7 @@ BucketIntegrityChecker::run(framework::ThreadHandle& thread) { while (!thread.interrupted()) { thread.registerTick(framework::PROCESS_CYCLE); - // Get the state based on the current time. + // Get the state based on the current time. framework::SecondTime currentTime( _component.getClock().getTimeInSeconds()); @@ -543,8 +514,7 @@ BucketIntegrityChecker::run(framework::ThreadHandle& thread) (_scheduleOptions._requestDelay - (currentTime - _lastResponseTime)).getMillis()); if (delay > _maxThreadWaitTime) delay = _maxThreadWaitTime; - monitor.wait(std::min(_maxThreadWaitTime.getTime(), - delay.getTime())); + monitor.wait(std::min(_maxThreadWaitTime.getTime(), delay.getTime())); thread.registerTick(framework::WAIT_CYCLE); } else if (_lastCycleCompleted && getTotalPendingCount() > 0) { LOG(spam, "Completed last cycle. Waiting until we have 0 pending " @@ -558,31 +528,29 @@ BucketIntegrityChecker::run(framework::ThreadHandle& thread) _scheduleOptions._maxPendingCount); // Else we send up to max pending and wait for responses. if (_lastCycleCompleted) { - for (uint32_t i=0; i<_status.size(); ++i) { - _status[i].state = DiskData::NOT_STARTED; - _status[i].failedRepairs.clear(); - _status[i].checkedBuckets = 0; - _status[i].retriedBuckets = 0; + for (auto& disk : _status) { + disk.state = DiskData::NOT_STARTED; + disk.failedRepairs.clear(); + disk.checkedBuckets = 0; + disk.retriedBuckets = 0; } LOG(info, "Starting new verification/repair cycle at time %s.", currentTime.toString().c_str()); _lastCycleStart = currentTime; - _cycleStartBucketCount = _component.getBucketDatabase(BucketSpace::placeHolder()).size(); + _cycleStartBucketCount = 0; + for (auto space : _bucketSpaces) { + _cycleStartBucketCount += _component.getBucketDatabase(space).size(); + } _lastCycleCompleted = false; - _currentRunWithFullVerification - = (state == SchedulingOptions::RUN_FULL); + _currentRunWithFullVerification = (state == SchedulingOptions::RUN_FULL); ++_cycleCount; } - for (uint32_t i=0; i<_status.size(); ++i) { - while (_status[i].pendingCount - < _scheduleOptions._maxPendingCount) - { - document::BucketId bid(_status[i].iterate( - _component.getBucketDatabase(BucketSpace::placeHolder()))); - if (bid == document::BucketId(0, 0)) { - LOG(debug, "Completed repair cycle for disk %u.", i); - // If there is no next bucket, we might have completed - // run + for (auto& disk : _status) { + while (disk.pendingCount < _scheduleOptions._maxPendingCount) { + auto bucket = disk.iterate(_bucketSpaces, _component.getBucketSpaceRepo()); + if (bucket.getBucketId() == document::BucketId(0, 0)) { + LOG(debug, "Completed repair cycle for disk %u.", disk.disk); + // If there is no next bucket, we might have completed run bool completed = true; for (uint32_t j=0; j<_status.size(); ++j) { if (!_status[j].done()) { @@ -596,17 +564,15 @@ BucketIntegrityChecker::run(framework::ThreadHandle& thread) } break; } - document::Bucket bucket(BucketSpace::placeHolder(), bid); - std::shared_ptr<RepairBucketCommand> cmd( - new RepairBucketCommand(bucket, _status[i].disk)); + auto cmd = std::make_shared<RepairBucketCommand>(bucket, disk.disk); cmd->verifyBody(_currentRunWithFullVerification); cmd->moveToIdealDisk(true); cmd->setPriority(230); LOG(spam, "Sending new repair command for bucket %s. " "After this, there will be %u pending on disk %u", - bid.toString().c_str(), - _status[i].pendingCount + 1, _status[i].disk); - ++_status[i].pendingCount; + bucket.toString().c_str(), + disk.pendingCount + 1, disk.disk); + ++disk.pendingCount; dispatchDown(cmd); } } @@ -620,8 +586,8 @@ uint32_t BucketIntegrityChecker::getTotalPendingCount() const { uint32_t total = 0; - for (uint32_t i=0; i<_status.size(); ++i) { - total += _status[i].pendingCount; + for (auto& disk : _status) { + total += disk.pendingCount; } return total; } diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h index b169090ab3c..fb619a84c46 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h @@ -10,6 +10,7 @@ #pragma once +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/storage/config/config-stor-integritychecker.h> @@ -67,17 +68,20 @@ public: */ enum State { NOT_STARTED, IN_PROGRESS, DONE }; + size_t currentBucketSpaceIndex; document::BucketId currentBucket; uint32_t pendingCount; State state; uint8_t disk; - std::list<document::BucketId> failedRepairs; + std::list<document::Bucket> failedRepairs; uint32_t checkedBuckets; uint32_t retriedBuckets; - DiskData() : currentBucket(0), pendingCount(0), - state(NOT_STARTED), disk(255), - checkedBuckets(0), retriedBuckets(0) {} + DiskData() + : currentBucketSpaceIndex(0), currentBucket(0), + pendingCount(0), state(NOT_STARTED), disk(255), + checkedBuckets(0), retriedBuckets(0) + {} bool done() const; // Whether we're still working on this disk bool working() const; // Whether we've stated and not finished @@ -85,11 +89,14 @@ public: * Get the next bucket to repair. If no more to iterate, random bucket * is returned. Check if done() afterwards. */ - document::BucketId iterate(StorBucketDatabase&); + document::Bucket iterate(const ContentBucketSpaceRepo::BucketSpaces& bucketSpaces, + const ContentBucketSpaceRepo& bucketSpaceRepo); }; private: + ServiceLayerComponent _component; uint32_t _cycleCount; + ContentBucketSpaceRepo::BucketSpaces _bucketSpaces; std::vector<DiskData> _status; framework::SecondTime _lastCycleStart; uint32_t _cycleStartBucketCount; @@ -98,11 +105,9 @@ private: bool _currentRunWithFullVerification; bool _verifyAllRepairs; SchedulingOptions _scheduleOptions; - lib::ClusterState _systemState; vespalib::Monitor _wait; config::ConfigFetcher _configFetcher; framework::MilliSecTime _maxThreadWaitTime; - ServiceLayerComponent _component; framework::Thread::UP _thread; BucketIntegrityChecker(const BucketIntegrityChecker &); @@ -130,7 +135,6 @@ private: void configure(std::unique_ptr<vespa::config::content::core::StorIntegritycheckerConfig>) override; void onDoneInit() override; bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; - bool onSetSystemState(const std::shared_ptr<api::SetSystemStateCommand>&) override; bool onNotifyBucketChangeReply(const std::shared_ptr<api::NotifyBucketChangeReply>&) override { return true; } SchedulingOptions::RunState getCurrentRunState(framework::SecondTime time) const; void run(framework::ThreadHandle&) override; diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 4b624199459..1a8be145470 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -5,6 +5,7 @@ #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/messagebucket.h> #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/bufferedlogger.h> @@ -20,10 +21,9 @@ ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( _metrics(), _configFetcher(configUri.getContext()), _stateLock(), - _currentDistribution(_component.getDistribution()), _currentState(), // Not set yet, so ownership will not be valid _currentOwnership(std::make_shared<OwnershipState>( - _currentDistribution, _currentState)), + _component.getBucketSpaceRepo(), _currentState)), _abortQueuedAndPendingOnStateChange(false), _abortMutatingIdealStateOps(false), _abortMutatingExternalLoadOps(false) @@ -66,7 +66,7 @@ ChangedBucketOwnershipHandler::setCurrentOwnershipWithStateNoLock( { _currentState = std::make_shared<lib::ClusterState>(newState); _currentOwnership = std::make_shared<OwnershipState>( - _currentDistribution, _currentState); + _component.getBucketSpaceRepo(), _currentState); } namespace { @@ -94,21 +94,32 @@ ChangedBucketOwnershipHandler::Metrics::Metrics(metrics::MetricSet* owner) {} ChangedBucketOwnershipHandler::Metrics::~Metrics() { } -ChangedBucketOwnershipHandler::OwnershipState::OwnershipState(const lib::Distribution::SP& distribution, +ChangedBucketOwnershipHandler::OwnershipState::OwnershipState(const ContentBucketSpaceRepo &contentBucketSpaceRepo, const lib::ClusterState::CSP& state) - : _distribution(distribution), + : _distributions(), _state(state) { + for (const auto &elem : contentBucketSpaceRepo) { + auto distribution = elem.second->getDistribution(); + if (distribution) { + _distributions.emplace(elem.first, std::move(distribution)); + } + } } + + ChangedBucketOwnershipHandler::OwnershipState::~OwnershipState() {} uint16_t ChangedBucketOwnershipHandler::OwnershipState::ownerOf( - const document::BucketId& bucket) const + const document::Bucket& bucket) const { + auto distributionItr = _distributions.find(bucket.getBucketSpace()); + assert(distributionItr != _distributions.end()); + const auto &distribution = *distributionItr->second; try { - return _distribution->getIdealDistributorNode(*_state, bucket); + return distribution.getIdealDistributorNode(*_state, bucket.getBucketId()); } catch (lib::TooFewBucketBitsInUseException& e) { LOGBP(debug, "Too few bucket bits used for %s to be assigned to " @@ -121,7 +132,7 @@ ChangedBucketOwnershipHandler::OwnershipState::ownerOf( "for available distributors before reaching this code path! " "Cluster state is '%s', distribution is '%s'", _state->toString().c_str(), - _distribution->toString().c_str()); + distribution.toString().c_str()); } catch (const std::exception& e) { LOG(error, "Got unknown exception while resolving distributor: %s", @@ -159,8 +170,8 @@ class StateDiffLazyAbortPredicate if (_allDistributorsHaveGoneDown) { return true; } - uint16_t oldOwner(_oldState.ownerOf(bucket.getBucketId())); - uint16_t newOwner(_newState.ownerOf(bucket.getBucketId())); + uint16_t oldOwner(_oldState.ownerOf(bucket)); + uint16_t newOwner(_newState.ownerOf(bucket)); if (oldOwner != newOwner) { LOG(spam, "Owner of %s was %u, now %u. Operation should be aborted", bucket.toString().c_str(), oldOwner, newOwner); @@ -262,9 +273,8 @@ void ChangedBucketOwnershipHandler::storageDistributionChanged() { vespalib::LockGuard guard(_stateLock); - _currentDistribution = _component.getDistribution(); _currentOwnership = std::make_shared<OwnershipState>( - _currentDistribution, _currentState); + _component.getBucketSpaceRepo(), _currentState); } bool @@ -324,7 +334,7 @@ ChangedBucketOwnershipHandler::sendingDistributorOwnsBucketInCurrentState( try { document::Bucket opBucket(getStorageMessageBucket(cmd)); - return (current->ownerOf(opBucket.getBucketId()) == cmd.getSourceIndex()); + return (current->ownerOf(opBucket) == cmd.getSourceIndex()); } catch (vespalib::IllegalArgumentException& e) { LOG(error, "Precondition violation: unable to get bucket from " diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h index b9508fc91b2..9c6e4256db6 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h @@ -13,6 +13,7 @@ #include <vespa/storage/persistence/messages.h> #include <atomic> #include <vector> +#include <unordered_map> namespace storage { @@ -63,7 +64,7 @@ public: }; /** - * Wrapper around the distribution & state pair that decides how to + * Wrapper around the distribution & state pairs that decides how to * compute the owner distributor for a bucket. It's possible to have * an ownership state with a nullptr cluster state when the node * initially starts up, which is why no owership state must be used unless @@ -71,21 +72,21 @@ public: */ class OwnershipState { - lib::Distribution::SP _distribution; + using BucketSpace = document::BucketSpace; + std::unordered_map<BucketSpace, std::shared_ptr<const lib::Distribution>, BucketSpace::hash> _distributions; lib::ClusterState::CSP _state; public: using SP = std::shared_ptr<OwnershipState>; using CSP = std::shared_ptr<const OwnershipState>; - OwnershipState(const lib::Distribution::SP& distribution, + OwnershipState(const ContentBucketSpaceRepo &contentBucketSpaceRepo, const lib::ClusterState::CSP& state); ~OwnershipState(); static const uint16_t FAILED_TO_RESOLVE = 0xffff; bool valid() const { - return ((_distribution.get() != nullptr) - && (_state.get() != nullptr)); + return (!_distributions.empty() && _state); } /** @@ -96,7 +97,7 @@ public: return *_state; } - uint16_t ownerOf(const document::BucketId& bucket) const; + uint16_t ownerOf(const document::Bucket& bucket) const; }; /** @@ -111,7 +112,6 @@ private: Metrics _metrics; config::ConfigFetcher _configFetcher; vespalib::Lock _stateLock; - lib::Distribution::SP _currentDistribution; lib::ClusterState::CSP _currentState; OwnershipState::CSP _currentOwnership; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index c90b18038c2..eae51b90165 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -1,18 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "communicationmanager.h" #include "fnetlistener.h" #include "rpcrequestwrapper.h" -#include <vespa/storage/config/config-stor-server.h> -#include <vespa/storage/common/nodestateupdater.h> -#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> +#include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/storage/common/bucket_resolver.h> +#include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/config/config-stor-server.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".communication.manager"); @@ -267,6 +269,22 @@ CommunicationManager::handleReply(std::unique_ptr<mbus::Reply> reply) } } +namespace { + +struct PlaceHolderBucketResolver : public BucketResolver { + virtual document::Bucket bucketFromId(const document::DocumentId &) const override { + return document::Bucket(document::BucketSpace::placeHolder(), document::BucketId(0)); + } + virtual document::BucketSpace bucketSpaceFromName(const vespalib::string &) const override { + return document::BucketSpace::placeHolder(); + } + virtual vespalib::string nameFromBucketSpace(const document::BucketSpace &) const override { + return ""; + } +}; + +} + CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri) : StorageLink("Communication manager"), _component(compReg, "communicationmanager"), @@ -277,7 +295,8 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co _count(0), _configUri(configUri), _closed(false), - _docApiConverter(configUri), + _bucketResolver(std::make_unique<PlaceHolderBucketResolver>()), + _docApiConverter(configUri, *_bucketResolver), _messageAllocTypes(_component.getMemoryManager()) { _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index e1f7888ac67..4cf3f33e6ea 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -35,6 +35,7 @@ namespace mbus { } namespace storage { +class BucketResolver; class VisitorMbusSession; class Visitor; class VisitorThread; @@ -167,6 +168,7 @@ private: config::ConfigUri _configUri; std::atomic<bool> _closed; + std::unique_ptr<BucketResolver> _bucketResolver; DocumentApiConverter _docApiConverter; framework::Thread::UP _thread; MessageAllocationTypes _messageAllocTypes; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index ddc11e9ad77..c2761b3d832 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -1,18 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "documentapiconverter.h" #include "priorityconverter.h" +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/documentapi/documentapi.h> -#include <vespa/storageapi/message/visitor.h> +#include <vespa/storage/common/bucket_resolver.h> +#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/datagram.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/searchresult.h> -#include <vespa/storageapi/message/queryresult.h> #include <vespa/storageapi/message/documentsummary.h> #include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/queryresult.h> #include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/searchresult.h> #include <vespa/storageapi/message/stat.h> -#include <vespa/storageapi/message/batch.h> -#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/storageapi/message/visitor.h> #include <vespa/log/log.h> LOG_SETUP(".documentapiconverter"); @@ -21,8 +23,10 @@ using document::BucketSpace; namespace storage { -DocumentApiConverter::DocumentApiConverter(const config::ConfigUri & configUri) - : _priConverter(std::make_unique<PriorityConverter>(configUri)) +DocumentApiConverter::DocumentApiConverter(const config::ConfigUri &configUri, + const BucketResolver &bucketResolver) + : _priConverter(std::make_unique<PriorityConverter>(configUri)), + _bucketResolver(bucketResolver) {} DocumentApiConverter::~DocumentApiConverter() {} @@ -38,7 +42,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_PUTDOCUMENT: { documentapi::PutDocumentMessage& from(static_cast<documentapi::PutDocumentMessage&>(fromMsg)); - auto to = std::make_unique<api::PutCommand>(document::Bucket(BucketSpace::placeHolder(), document::BucketId(0)), from.stealDocument(), from.getTimestamp()); + document::Bucket bucket = _bucketResolver.bucketFromId(from.getDocument().getId()); + auto to = std::make_unique<api::PutCommand>(bucket, from.stealDocument(), from.getTimestamp()); to->setCondition(from.getCondition()); toMsg = std::move(to); break; @@ -46,8 +51,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: { documentapi::UpdateDocumentMessage& from(static_cast<documentapi::UpdateDocumentMessage&>(fromMsg)); - auto to = std::make_unique<api::UpdateCommand>(document::Bucket(BucketSpace::placeHolder(), document::BucketId(0)), from.stealDocumentUpdate(), - from.getNewTimestamp()); + document::Bucket bucket = _bucketResolver.bucketFromId(from.getDocumentUpdate().getId()); + auto to = std::make_unique<api::UpdateCommand>(bucket, from.stealDocumentUpdate(), from.getNewTimestamp()); to->setOldTimestamp(from.getOldTimestamp()); to->setCondition(from.getCondition()); toMsg = std::move(to); @@ -56,7 +61,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: { documentapi::RemoveDocumentMessage& from(static_cast<documentapi::RemoveDocumentMessage&>(fromMsg)); - auto to = std::make_unique<api::RemoveCommand>(document::Bucket(BucketSpace::placeHolder(), document::BucketId(0)), from.getDocumentId(), 0); + auto to = std::make_unique<api::RemoveCommand>(_bucketResolver.bucketFromId(from.getDocumentId()), from.getDocumentId(), 0); to->setCondition(from.getCondition()); toMsg = std::move(to); break; @@ -64,14 +69,14 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_GETDOCUMENT: { documentapi::GetDocumentMessage& from(static_cast<documentapi::GetDocumentMessage&>(fromMsg)); - auto to = std::make_unique<api::GetCommand>(document::Bucket(BucketSpace::placeHolder(), document::BucketId(0)), from.getDocumentId(), from.getFieldSet()); + auto to = std::make_unique<api::GetCommand>(_bucketResolver.bucketFromId(from.getDocumentId()), from.getDocumentId(), from.getFieldSet()); toMsg.reset(to.release()); break; } case DocumentProtocol::MESSAGE_CREATEVISITOR: { documentapi::CreateVisitorMessage& from(static_cast<documentapi::CreateVisitorMessage&>(fromMsg)); - auto to = std::make_unique<api::CreateVisitorCommand>(BucketSpace::placeHolder(), + auto to = std::make_unique<api::CreateVisitorCommand>(_bucketResolver.bucketSpaceFromName(from.getBucketSpace()), from.getLibraryName(), from.getInstanceId(), from.getDocumentSelection()); @@ -113,13 +118,15 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_STATBUCKET: { documentapi::StatBucketMessage& from(static_cast<documentapi::StatBucketMessage&>(fromMsg)); - toMsg = std::make_unique<api::StatBucketCommand>(from.getBucket(), from.getDocumentSelection()); + document::Bucket bucket(_bucketResolver.bucketSpaceFromName(from.getBucketSpace()), from.getBucketId()); + toMsg = std::make_unique<api::StatBucketCommand>(bucket, from.getDocumentSelection()); break; } case DocumentProtocol::MESSAGE_GETBUCKETLIST: { documentapi::GetBucketListMessage& from(static_cast<documentapi::GetBucketListMessage&>(fromMsg)); - toMsg = std::make_unique<api::GetBucketListCommand>(from.getBucket()); + document::Bucket bucket(_bucketResolver.bucketSpaceFromName(from.getBucketSpace()), from.getBucketId()); + toMsg = std::make_unique<api::GetBucketListCommand>(bucket); break; } case DocumentProtocol::MESSAGE_VISITORINFO: @@ -138,7 +145,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_REMOVELOCATION: { documentapi::RemoveLocationMessage& from(static_cast<documentapi::RemoveLocationMessage&>(fromMsg)); - api::RemoveLocationCommand::UP to(new api::RemoveLocationCommand(from.getDocumentSelection(), document::Bucket(BucketSpace::placeHolder(), document::BucketId(0)))); + document::Bucket bucket(_bucketResolver.bucketSpaceFromName(from.getBucketSpace()), document::BucketId(0)); + api::RemoveLocationCommand::UP to(new api::RemoveLocationCommand(from.getDocumentSelection(), bucket)); toMsg.reset(to.release()); break; } @@ -290,6 +298,7 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg, const document documentapi::CreateVisitorMessage::UP to( new documentapi::CreateVisitorMessage(from.getLibraryName(), from.getInstanceId(), from.getControlDestination(), from.getDataDestination())); + to->setBucketSpace(_bucketResolver.nameFromBucketSpace(from.getBucketSpace())); to->setDocumentSelection(from.getDocumentSelection()); to->setMaximumPendingReplyCount(from.getMaximumPendingReplyCount()); to->setParameters(from.getParameters()); @@ -315,7 +324,9 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg, const document case api::MessageType::STATBUCKET_ID: { api::StatBucketCommand& from(static_cast<api::StatBucketCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::StatBucketMessage>(from.getBucket(), from.getDocumentSelection()); + auto statMsg = std::make_unique<documentapi::StatBucketMessage>(from.getBucket().getBucketId(), from.getDocumentSelection()); + statMsg->setBucketSpace(_bucketResolver.nameFromBucketSpace(from.getBucket().getBucketSpace())); + toMsg = std::move(statMsg); break; } default: diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.h b/storage/src/vespa/storage/storageserver/documentapiconverter.h index 0cc2f3f3b9c..5310bcd0127 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.h +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.h @@ -13,6 +13,7 @@ namespace api { class StorageReply; } +class BucketResolver; class PriorityConverter; /** Converts messages from storageapi to documentapi and @@ -21,7 +22,8 @@ class PriorityConverter; class DocumentApiConverter { public: - DocumentApiConverter(const config::ConfigUri & configUri); + DocumentApiConverter(const config::ConfigUri &configUri, + const BucketResolver &bucketResolver); ~DocumentApiConverter(); std::unique_ptr<api::StorageCommand> toStorageAPI(documentapi::DocumentMessage& msg, const document::DocumentTypeRepo::SP &repo); @@ -31,6 +33,7 @@ public: const PriorityConverter& getPriorityConverter() const { return *_priConverter; } private: std::unique_ptr<PriorityConverter> _priConverter; + const BucketResolver &_bucketResolver; }; } // namespace storage diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index be30c459bdf..60dedab5ce8 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -431,7 +431,7 @@ bool MergeThrottler::isMergeAlreadyKnown(const api::StorageMessage::SP& msg) const { auto& mergeCmd = static_cast<const api::MergeBucketCommand&>(*msg); - return _merges.find(mergeCmd.getBucketId()) != _merges.end(); + return _merges.find(mergeCmd.getBucket()) != _merges.end(); } bool @@ -830,10 +830,8 @@ MergeThrottler::processNewMergeCommand( // and that we can fit it into our window. // Register the merge now so that it will contribute to filling up our // merge throttling window. - assert(_merges.find(mergeCmd.getBucketId()) == _merges.end()); - auto state = _merges.insert( - std::make_pair(mergeCmd.getBucketId(), - ChainedMergeState(msg))).first; + assert(_merges.find(mergeCmd.getBucket()) == _merges.end()); + auto state = _merges.emplace(mergeCmd.getBucket(), ChainedMergeState(msg)).first; LOG(debug, "Added merge %s to internal state", mergeCmd.toString().c_str()); @@ -911,7 +909,7 @@ MergeThrottler::processCycledMergeCommand( MergeNodeSequence nodeSeq(mergeCmd, _component.getIndex()); - auto mergeIter = _merges.find(mergeCmd.getBucketId()); + auto mergeIter = _merges.find(mergeCmd.getBucket()); assert(mergeIter != _merges.end()); if (mergeIter->second.isAborted()) { @@ -964,7 +962,7 @@ MergeThrottler::processMergeReply( { auto& mergeReply = dynamic_cast<const api::MergeBucketReply&>(*msg); - auto mergeIter = _merges.find(mergeReply.getBucketId()); + auto mergeIter = _merges.find(mergeReply.getBucket()); if (mergeIter == _merges.end()) { LOG(warning, "Received %s, which has no command mapped " "for it. Cannot send chained reply!", @@ -1075,7 +1073,7 @@ MergeThrottler::onDown(const std::shared_ptr<api::StorageMessage>& msg) } else if (isDiffCommand(*msg)) { vespalib::LockGuard lock(_stateLock); auto& cmd = static_cast<api::StorageCommand&>(*msg); - if (bucketIsUnknownOrAborted(cmd.getBucketId())) { + if (bucketIsUnknownOrAborted(cmd.getBucket())) { sendUp(makeAbortReply(cmd, "no state recorded for bucket in merge " "throttler, source merge probably aborted earlier")); return true; @@ -1104,7 +1102,7 @@ MergeThrottler::isMergeReply(const api::StorageMessage& msg) const } bool -MergeThrottler::bucketIsUnknownOrAborted(const document::BucketId& bucket) const +MergeThrottler::bucketIsUnknownOrAborted(const document::Bucket& bucket) const { auto it = _merges.find(bucket); if (it == _merges.end()) { diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 69fdfdc1b95..d62e9a042b2 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -13,7 +13,7 @@ #include <vespa/storage/distributor/messageguard.h> #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucket.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/messagebus/staticthrottlepolicy.h> #include <vespa/metrics/metrics.h> @@ -134,7 +134,7 @@ private: const std::string& getMergeCmdString() const { return _cmdString; } }; - typedef std::map<document::BucketId, ChainedMergeState> ActiveMergeMap; + typedef std::map<document::Bucket, ChainedMergeState> ActiveMergeMap; // Use a set rather than a priority_queue, since we want to be // able to iterate over the collection during status rendering @@ -371,7 +371,7 @@ private: bool isDiffCommand(const api::StorageMessage& msg) const; bool isMergeCommand(const api::StorageMessage& msg) const; bool isMergeReply(const api::StorageMessage& msg) const; - bool bucketIsUnknownOrAborted(const document::BucketId& bucket) const; + bool bucketIsUnknownOrAborted(const document::Bucket& bucket) const; std::shared_ptr<api::StorageMessage> makeAbortReply( api::StorageCommand& cmd, diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index cb0cf756586..a04b0d71e96 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -46,6 +46,8 @@ struct StorageProtocolTest : public CppUnit::TestFixture { vespalib::Version _version5_0{5, 0, 12}; vespalib::Version _version5_1{5, 1, 0}; vespalib::Version _version5_2{5, 93, 30}; + // TODO: Set correct version when bucket space serialization is activated by default + vespalib::Version _version6_0{6, 999, 0}; documentapi::LoadTypeSet _loadTypes; mbusprot::StorageProtocol _protocol; static std::vector<std::string> _nonVerboseMessageStrings; @@ -58,7 +60,7 @@ struct StorageProtocolTest : public CppUnit::TestFixture { _testDoc(_docMan.createDocument()), _testDocId(_testDoc->getId()), _bucket(makeDocumentBucket(document::BucketId(16, 0x51))), - _protocol(_docMan.getTypeRepoSP(), _loadTypes) + _protocol(_docMan.getTypeRepoSP(), _loadTypes, true) { _loadTypes.addLoadType(34, "foo", documentapi::Priority::PRI_NORMAL_2); } @@ -102,6 +104,10 @@ struct StorageProtocolTest : public CppUnit::TestFixture { void testUpdateCommand52(); void testRemoveCommand52(); + void testPutCommandWithBucketSpace6_0(); + void testCreateVisitorWithBucketSpace6_0(); + void testRequestBucketInfoWithBucketSpace6_0(); + CPPUNIT_TEST_SUITE(StorageProtocolTest); // Enable to see string outputs of messages @@ -139,6 +145,11 @@ struct StorageProtocolTest : public CppUnit::TestFixture { CPPUNIT_TEST(testUpdateCommand52); CPPUNIT_TEST(testRemoveCommand52); + // 6.0 tests + CPPUNIT_TEST(testPutCommandWithBucketSpace6_0); + CPPUNIT_TEST(testCreateVisitorWithBucketSpace6_0); + CPPUNIT_TEST(testRequestBucketInfoWithBucketSpace6_0); + CPPUNIT_TEST_SUITE_END(); }; @@ -947,6 +958,44 @@ StorageProtocolTest::testRemoveCommand52() } void +StorageProtocolTest::testPutCommandWithBucketSpace6_0() +{ + ScopedName test("testPutCommandWithBucketSpace6_0"); + + document::Bucket bucket(document::BucketSpace(5), _bucket.getBucketId()); + auto cmd = std::make_shared<PutCommand>(bucket, _testDoc, 14); + + auto cmd2 = copyCommand(cmd, _version6_0); + CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucket()); +} + +void +StorageProtocolTest::testCreateVisitorWithBucketSpace6_0() +{ + ScopedName test("testCreateVisitorWithBucketSpace6_0"); + + document::BucketSpace bucketSpace(5); + auto cmd = std::make_shared<CreateVisitorCommand>(bucketSpace, "library", "id", "doc selection"); + + auto cmd2 = copyCommand(cmd, _version6_0); + CPPUNIT_ASSERT_EQUAL(bucketSpace, cmd2->getBucketSpace()); +} + +void +StorageProtocolTest::testRequestBucketInfoWithBucketSpace6_0() +{ + ScopedName test("testRequestBucketInfoWithBucketSpace6_0"); + + document::BucketSpace bucketSpace(5); + std::vector<document::BucketId> ids = {document::BucketId(3)}; + auto cmd = std::make_shared<RequestBucketInfoCommand>(bucketSpace, ids); + + auto cmd2 = copyCommand(cmd, _version6_0); + CPPUNIT_ASSERT_EQUAL(bucketSpace, cmd2->getBucketSpace()); + CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); +} + +void StorageProtocolTest::testStringOutputs() { std::cerr << "\nNon verbose output:\n"; diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index c59dd83663d..d5952d7cb91 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -10,5 +10,6 @@ vespa_add_library(storageapi_mbusprot OBJECT protocolserialization5_0.cpp protocolserialization5_1.cpp protocolserialization5_2.cpp + protocolserialization6_0.cpp DEPENDS ) diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h index e32942d0303..d1e5783e609 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/document/bucket/bucket.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/messagebus/routable.h> #include <vespa/storageapi/mbusprot/storagemessage.h> @@ -176,6 +177,10 @@ protected: virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const = 0; virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const = 0; + virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; + virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; + virtual document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const = 0; + virtual void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const = 0; virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index e4993a1ee7b..cd8e4992ba5 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -41,7 +41,7 @@ void ProtocolSerialization4_2::onEncode( char* pos = buf.allocate(docBlockSize); vdslib::DocumentList copy(msg.getOperations(), pos, docBlockSize); buf.putBoolean(msg.keepTimeStamps()); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); } @@ -52,8 +52,7 @@ ProtocolSerialization4_2::onDecodeMultiOperationCommand(BBuf& buf) const std::vector<char> buffer(length); buf.getBytes(&buffer[0], length); bool keepTimestamps = SH::getBoolean(buf); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::MultiOperationCommand::UP msg( new api::MultiOperationCommand(getTypeRepoSp(), bucket, buffer, keepTimestamps)); @@ -67,7 +66,7 @@ ProtocolSerialization4_2::onEncode( { // Serialization format - allow different types of serialization depending on source. buf.putByte(0); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putInt(msg.getOperationCount()); for (uint32_t i = 0; i < msg.getOperationCount(); i++) { @@ -101,8 +100,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeBatchPutRemoveCommand(BBuf& buf) const { SH::getByte(buf); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); std::unique_ptr<api::BatchPutRemoveCommand> cmd(new api::BatchPutRemoveCommand(bucket)); int length = SH::getInt(buf); @@ -164,7 +162,7 @@ void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putLong(msg.getBeforeTimestamp()); buf.putBoolean(msg.getFieldSet() == "[header]"); onEncodeCommand(buf, msg); @@ -174,8 +172,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const { document::DocumentId did(SH::getString(buf)); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); bool headerOnly(SH::getBoolean(buf)); api::GetCommand::UP msg( @@ -188,7 +185,7 @@ void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::RemoveCommand& msg) const { buf.putString(msg.getDocumentId().toString()); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putLong(msg.getTimestamp()); onEncodeBucketInfoCommand(buf, msg); } @@ -197,8 +194,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const { document::DocumentId did(SH::getString(buf)); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); api::RemoveCommand::UP msg(new api::RemoveCommand(bucket, did, timestamp)); onDecodeBucketInfoCommand(buf, *msg); @@ -208,7 +204,7 @@ ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::RevertCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putInt(msg.getRevertTokens().size()); for (uint32_t i=0, n=msg.getRevertTokens().size(); i<n; ++i) { buf.putLong(msg.getRevertTokens()[i]); @@ -219,8 +215,7 @@ void ProtocolSerialization4_2::onEncode( api::StorageCommand::UP ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); std::vector<api::Timestamp> tokens(SH::getInt(buf)); for (uint32_t i=0, n=tokens.size(); i<n; ++i) { tokens[i] = SH::getLong(buf); @@ -233,15 +228,14 @@ ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::CreateBucketCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); } api::StorageCommand::UP ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); onDecodeBucketInfoCommand(buf, *msg); return api::StorageCommand::UP(msg.release()); @@ -250,7 +244,7 @@ ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::MergeBucketCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); buf.putShort(nodes.size()); for (uint32_t i=0; i<nodes.size(); ++i) { @@ -265,8 +259,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const { typedef api::MergeBucketCommand::Node Node; - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); uint16_t nodeCount = SH::getShort(buf); std::vector<Node> nodes; nodes.reserve(nodeCount); @@ -285,7 +278,7 @@ ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::GetBucketDiffCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); buf.putShort(nodes.size()); for (uint32_t i=0; i<nodes.size(); ++i) { @@ -305,8 +298,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const { typedef api::MergeBucketCommand::Node Node; - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); uint16_t nodeCount = SH::getShort(buf); std::vector<Node> nodes; nodes.reserve(nodeCount); @@ -335,7 +327,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); buf.putShort(nodes.size()); for (uint32_t i=0; i<nodes.size(); ++i) { @@ -363,8 +355,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const { typedef api::MergeBucketCommand::Node Node; - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); uint16_t nodeCount = SH::getShort(buf); std::vector<Node> nodes; nodes.reserve(nodeCount); @@ -440,7 +431,7 @@ ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); putBucketInfo(msg.getBucketInfo(), buf); onEncodeCommand(buf, msg); } @@ -448,8 +439,7 @@ void ProtocolSerialization4_2::onEncode( api::StorageCommand::UP ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::BucketInfo info(getBucketInfo(buf)); api::NotifyBucketChangeCommand::UP msg( new api::NotifyBucketChangeCommand(bucket, info)); @@ -476,7 +466,7 @@ ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd, void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::SplitBucketCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putByte(msg.getMinSplitBits()); buf.putByte(msg.getMaxSplitBits()); buf.putInt(msg.getMinByteSize()); @@ -487,8 +477,7 @@ void ProtocolSerialization4_2::onEncode( api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::SplitBucketCommand::UP msg(new api::SplitBucketCommand(bucket)); msg->setMinSplitBits(SH::getByte(buf)); msg->setMaxSplitBits(SH::getByte(buf)); @@ -531,6 +520,7 @@ void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::CreateVisitorCommand& msg) const { + putBucketSpace(msg.getBucketSpace(), buf); buf.putString(msg.getLibraryName()); buf.putString(msg.getInstanceId()); buf.putString(msg.getDocumentSelection()); @@ -562,7 +552,7 @@ ProtocolSerialization4_2::onEncode( api::StorageCommand::UP ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const { - BucketSpace bucketSpace(BucketSpace::placeHolder()); + BucketSpace bucketSpace = getBucketSpace(buf); vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); @@ -638,7 +628,7 @@ void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const { buf.putString(msg.getDocumentSelection()); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); onEncodeCommand(buf, msg); } @@ -646,8 +636,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const { vespalib::stringref documentSelection = SH::getString(buf); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::RemoveLocationCommand::UP msg; msg.reset(new api::RemoveLocationCommand(documentSelection, bucket)); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index 22f58ebc58b..e5c8c490a53 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -12,6 +12,32 @@ using document::BucketSpace; namespace storage { namespace mbusprot { +document::Bucket +ProtocolSerialization5_0::getBucket(document::ByteBuffer& buf) const +{ + document::BucketId bucketId(SH::getLong(buf)); + return document::Bucket(BucketSpace::placeHolder(), bucketId); +} + +void +ProtocolSerialization5_0::putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const +{ + buf.putLong(bucket.getBucketId().getRawId()); + assert(bucket.getBucketSpace() == document::BucketSpace::placeHolder()); +} + +document::BucketSpace +ProtocolSerialization5_0::getBucketSpace(document::ByteBuffer&) const +{ + return BucketSpace::placeHolder(); +} + +void +ProtocolSerialization5_0::putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer&) const +{ + assert(bucketSpace == document::BucketSpace::placeHolder()); +} + api::BucketInfo ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const { @@ -93,7 +119,7 @@ void ProtocolSerialization5_0::onEncode( GBBuf& buf, const api::PutCommand& msg) const { SH::putDocument(msg.getDocument().get(), buf); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putLong(msg.getTimestamp()); buf.putLong(msg.getUpdateTimestamp()); onEncodeBucketInfoCommand(buf, msg); @@ -103,8 +129,7 @@ api::StorageCommand::UP ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const { document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::Timestamp ts(SH::getLong(buf)); api::PutCommand::UP msg(new api::PutCommand(bucket, doc, ts)); msg->setUpdateTimestamp(SH::getLong(buf)); @@ -203,7 +228,7 @@ void ProtocolSerialization5_0::onEncode( buf.putInt(0); } - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putLong(msg.getTimestamp()); buf.putLong(msg.getOldTimestamp()); onEncodeBucketInfoCommand(buf, msg); @@ -224,8 +249,7 @@ ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const SERIALIZE_HEAD)); } - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); api::UpdateCommand::UP msg( new api::UpdateCommand(bucket, update, timestamp)); @@ -270,7 +294,7 @@ void ProtocolSerialization5_0::onEncode( GBBuf& buf, const api::DeleteBucketCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); putBucketInfo(msg.getBucketInfo(), buf); } @@ -278,8 +302,7 @@ ProtocolSerialization5_0::onEncode( api::StorageCommand::UP ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::DeleteBucketCommand::UP msg(new api::DeleteBucketCommand(bucket)); onDecodeBucketInfoCommand(buf, *msg); if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) { @@ -495,7 +518,7 @@ void ProtocolSerialization5_0::onEncode( GBBuf& buf, const api::JoinBucketsCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putInt(msg.getSourceBuckets().size()); for (uint32_t i=0, n=msg.getSourceBuckets().size(); i<n; ++i) { buf.putLong(msg.getSourceBuckets()[i].getRawId()); @@ -507,8 +530,7 @@ ProtocolSerialization5_0::onEncode( api::StorageCommand::UP ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::JoinBucketsCommand::UP msg(new api::JoinBucketsCommand(bucket)); uint32_t size = SH::getInt(buf); if (size > buf.getRemaining()) { @@ -621,6 +643,7 @@ void ProtocolSerialization5_0::onEncode( for (uint32_t i=0; i<buckets.size(); ++i) { buf.putLong(buckets[i].getRawId()); } + putBucketSpace(msg.getBucketSpace(), buf); if (buckets.size() == 0) { buf.putShort(msg.getDistributor()); buf.putString(msg.getSystemState().toString()); @@ -638,7 +661,7 @@ ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const buckets[i] = document::BucketId(SH::getLong(buf)); } api::RequestBucketInfoCommand::UP msg; - BucketSpace bucketSpace(BucketSpace::placeHolder()); + BucketSpace bucketSpace = getBucketSpace(buf); if (buckets.size() != 0) { msg.reset(new api::RequestBucketInfoCommand(bucketSpace, buckets)); } else { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h index ff9f08d38a9..c1285939a1c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h @@ -15,6 +15,10 @@ public: ProtocolSerialization5_0(const document::DocumentTypeRepo::SP&, const documentapi::LoadTypeSet& loadTypes); + document::Bucket getBucket(document::ByteBuffer& buf) const override; + void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const override; + document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const override; + void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const override; api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const override; void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const override; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp index dc97742b733..0afdfebd5b7 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp @@ -64,7 +64,7 @@ ProtocolSerialization5_1::ProtocolSerialization5_1( void ProtocolSerialization5_1::onEncode( GBBuf& buf, const api::SetBucketStateCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putByte(static_cast<uint8_t>(msg.getState())); onEncodeCommand(buf, msg); } @@ -72,8 +72,7 @@ void ProtocolSerialization5_1::onEncode( api::StorageCommand::UP ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::SetBucketStateCommand::BUCKET_STATE state( static_cast<api::SetBucketStateCommand::BUCKET_STATE>( SH::getByte(buf))); @@ -103,7 +102,7 @@ void ProtocolSerialization5_1::onEncode( GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putLong(msg.getBeforeTimestamp()); buf.putString(msg.getFieldSet()); onEncodeCommand(buf, msg); @@ -113,8 +112,7 @@ api::StorageCommand::UP ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const { document::DocumentId did(SH::getString(buf)); - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); std::string fieldSet(SH::getString(buf)); api::GetCommand::UP msg( @@ -127,6 +125,7 @@ void ProtocolSerialization5_1::onEncode( GBBuf& buf, const api::CreateVisitorCommand& msg) const { + putBucketSpace(msg.getBucketSpace(), buf); buf.putString(msg.getLibraryName()); buf.putString(msg.getInstanceId()); buf.putString(msg.getDocumentSelection()); @@ -161,7 +160,7 @@ ProtocolSerialization5_1::onEncode( api::StorageCommand::UP ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const { - BucketSpace bucketSpace(BucketSpace::placeHolder()); + BucketSpace bucketSpace = getBucketSpace(buf); vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); @@ -208,7 +207,7 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const void ProtocolSerialization5_1::onEncode( GBBuf& buf, const api::CreateBucketCommand& msg) const { - buf.putLong(msg.getBucketId().getRawId()); + putBucket(msg.getBucket(), buf); buf.putBoolean(msg.getActive()); onEncodeBucketInfoCommand(buf, msg); } @@ -216,8 +215,7 @@ void ProtocolSerialization5_1::onEncode( api::StorageCommand::UP ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const { - document::BucketId bucketId(SH::getLong(buf)); - document::Bucket bucket(BucketSpace::placeHolder(), bucketId); + document::Bucket bucket = getBucket(buf); bool setActive = SH::getBoolean(buf); api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); msg->setActive(setActive); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp new file mode 100644 index 00000000000..7257b6bc284 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "protocolserialization6_0.h" +#include "serializationhelper.h" + +namespace storage { +namespace mbusprot { + +ProtocolSerialization6_0::ProtocolSerialization6_0(const document::DocumentTypeRepo::SP &repo, + const documentapi::LoadTypeSet &loadTypes) + : ProtocolSerialization5_2(repo, loadTypes) +{ +} + +document::Bucket +ProtocolSerialization6_0::getBucket(document::ByteBuffer &buf) const +{ + document::BucketSpace bucketSpace(SH::getLong(buf)); + document::BucketId bucketId(SH::getLong(buf)); + return document::Bucket(bucketSpace, bucketId); +} + +void +ProtocolSerialization6_0::putBucket(const document::Bucket &bucket, vespalib::GrowableByteBuffer &buf) const +{ + buf.putLong(bucket.getBucketSpace().getId()); + buf.putLong(bucket.getBucketId().getRawId()); +} + +document::BucketSpace +ProtocolSerialization6_0::getBucketSpace(document::ByteBuffer &buf) const +{ + return document::BucketSpace(SH::getLong(buf)); +} + +void +ProtocolSerialization6_0::putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer &buf) const +{ + buf.putLong(bucketSpace.getId()); +} + +} +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h new file mode 100644 index 00000000000..015ec33b32f --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "protocolserialization5_2.h" +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace storage { +namespace mbusprot { + +/** + * Protocol serialization version adding decoding and encoding + * of bucket space to almost all commands. + */ +class ProtocolSerialization6_0 : public ProtocolSerialization5_2 +{ +public: + ProtocolSerialization6_0(const document::DocumentTypeRepo::SP &repo, + const documentapi::LoadTypeSet &loadTypes); + + document::Bucket getBucket(document::ByteBuffer &buf) const override; + void putBucket(const document::Bucket &bucket, vespalib::GrowableByteBuffer &buf) const override; + document::BucketSpace getBucketSpace(document::ByteBuffer &buf) const override; + void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer &buf) const override; +}; + +} +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 70be64800ca..edc8ca6a3b6 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -15,10 +15,13 @@ namespace storage::mbusprot { mbus::string StorageProtocol::NAME = "StorageProtocol"; StorageProtocol::StorageProtocol(const document::DocumentTypeRepo::SP repo, - const documentapi::LoadTypeSet& loadTypes) + const documentapi::LoadTypeSet& loadTypes, + bool activateBucketSpaceSerialization) : _serializer5_0(repo, loadTypes), _serializer5_1(repo, loadTypes), - _serializer5_2(repo, loadTypes) + _serializer5_2(repo, loadTypes), + _serializer6_0(repo, loadTypes), + _activateBucketSpaceSerialization(activateBucketSpaceSerialization) { } @@ -31,6 +34,8 @@ StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const } namespace { + // TODO: Set correct version when bucket space serialization is activated by default + vespalib::Version version6_0(6, 999, 0); vespalib::Version version5_2(5, 93, 30); vespalib::Version version5_1(5, 1, 0); vespalib::Version version5_0(5, 0, 12); @@ -93,7 +98,15 @@ StorageProtocol::encode(const vespalib::Version& version, } else if (version < version5_2) { return encodeMessage(_serializer5_1, routable, message, version5_1, version); } else { - return encodeMessage(_serializer5_2, routable, message, version5_2, version); + if (!_activateBucketSpaceSerialization) { + return encodeMessage(_serializer5_2, routable, message, version5_2, version); + } else { + if (version < version6_0) { + return encodeMessage(_serializer5_2, routable, message, version5_2, version); + } else { + return encodeMessage(_serializer6_0, routable, message, version6_0, version); + } + } } } catch (std::exception & e) { @@ -156,7 +169,15 @@ StorageProtocol::decode(const vespalib::Version & version, } else if (version < version5_2) { return decodeMessage(_serializer5_1, data, type, version5_1, version); } else { - return decodeMessage(_serializer5_2, data, type, version5_2, version); + if (!_activateBucketSpaceSerialization) { + return decodeMessage(_serializer5_2, data, type, version5_2, version); + } else { + if (version < version6_0) { + return decodeMessage(_serializer5_2, data, type, version5_2, version); + } else { + return decodeMessage(_serializer6_0, data, type, version6_0, version); + } + } } } catch (std::exception & e) { std::ostringstream ost; diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h index 10289adaf1a..437be5dcbe6 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -2,6 +2,7 @@ #pragma once #include "protocolserialization5_2.h" +#include "protocolserialization6_0.h" #include <vespa/messagebus/iprotocol.h> namespace storage::mbusprot { @@ -13,7 +14,9 @@ public: static mbus::string NAME; - StorageProtocol(const document::DocumentTypeRepo::SP, const documentapi::LoadTypeSet& loadTypes); + StorageProtocol(const document::DocumentTypeRepo::SP, + const documentapi::LoadTypeSet& loadTypes, + bool activateBucketSpaceSerialization = false); ~StorageProtocol(); const mbus::string& getName() const override { return NAME; } @@ -25,6 +28,8 @@ private: ProtocolSerialization5_0 _serializer5_0; ProtocolSerialization5_1 _serializer5_1; ProtocolSerialization5_2 _serializer5_2; + ProtocolSerialization6_0 _serializer6_0; + bool _activateBucketSpaceSerialization; }; } diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index 18ad95c2c02..0961a8f6edc 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -475,6 +475,12 @@ RequestBucketInfoCommand::RequestBucketInfoCommand( { } +document::Bucket +RequestBucketInfoCommand::getBucket() const +{ + return document::Bucket(_bucketSpace, document::BucketId()); +} + void RequestBucketInfoCommand::print(std::ostream& out, bool verbose, const std::string& indent) const diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index 05838600a24..5fba1a3bf65 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -363,6 +363,7 @@ public: const vespalib::string& getDistributionHash() const { return _distributionHash; } document::BucketSpace getBucketSpace() const { return _bucketSpace; } + document::Bucket getBucket() const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index 7b5a614bd3e..3cb6f72d5d9 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -68,6 +68,12 @@ CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) CreateVisitorCommand::~CreateVisitorCommand() {} +document::Bucket +CreateVisitorCommand::getBucket() const +{ + return document::Bucket(_bucketSpace, document::BucketId()); +} + void CreateVisitorCommand::print(std::ostream& out, bool verbose, const std::string& indent) const diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h index f252ecd344f..e1850686222 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -86,6 +86,7 @@ public: VisitorId getVisitorId() const { return _visitorId; } uint32_t getVisitorCmdId() const { return _visitorCmdId; } document::BucketSpace getBucketSpace() const { return _bucketSpace; } + document::Bucket getBucket() const override; const vespalib::string & getLibraryName() const { return _libName; } const vespalib::string & getInstanceId() const { return _instanceId; } const vespalib::string & getControlDestination() const diff --git a/testutil/src/main/java/com/yahoo/test/ManualClock.java b/testutil/src/main/java/com/yahoo/test/ManualClock.java index 28ccdc8d27e..ffef6895c38 100644 --- a/testutil/src/main/java/com/yahoo/test/ManualClock.java +++ b/testutil/src/main/java/com/yahoo/test/ManualClock.java @@ -34,6 +34,10 @@ public class ManualClock extends Clock { currentTime = currentTime.plus(temporal); } + public void setInstant(Instant time) { + currentTime = time; + } + @Override public Instant instant() { return currentTime; } diff --git a/travis/travis-build-full.sh b/travis/travis-build-full.sh index 01333c9d0dd..fc0efb843aa 100755 --- a/travis/travis-build-full.sh +++ b/travis/travis-build-full.sh @@ -3,7 +3,7 @@ set -e export SOURCE_DIR=/source -export NUM_THREADS=4 +export NUM_THREADS=6 export MALLOC_ARENA_MAX=1 export MAVEN_OPTS="-Xms128m -Xmx2g" source /etc/profile.d/devtoolset-6.sh || true diff --git a/travis/travis.sh b/travis/travis.sh index 5b246b64fb9..77bfebf2505 100755 --- a/travis/travis.sh +++ b/travis/travis.sh @@ -5,7 +5,7 @@ set -e # Workaround for Travis log output timeout (jobs without output over 10 minutes are killed) function bell() { while true; do - echo -e "\a" + echo "." sleep 300 done } @@ -16,4 +16,3 @@ bell & docker run --rm -v ${HOME}/.m2:/root/.m2 -v ${HOME}/.ccache:/root/.ccache -v $(pwd):/source \ --entrypoint /source/travis/travis-build-full.sh ${DOCKER_IMAGE} exit $? - diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index db0fada990f..75607be3dfd 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -35,7 +35,6 @@ import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; import com.yahoo.vespa.http.client.config.SessionParams; -import org.apache.hadoop.mapreduce.v2.app.job.Task; /** * VespaRecordWriter sends the output <key, value> to one or more Vespa @@ -133,8 +132,8 @@ public class VespaRecordWriter extends RecordWriter { } private void initialize() { - if (!configuration.dryrun() && configuration.randomSartupSleepMs() > 0) { - int delay = new Random().nextInt(configuration.randomSartupSleepMs()); + if (!configuration.dryrun() && configuration.randomStartupSleepMs() > 0) { + int delay = new Random().nextInt(configuration.randomStartupSleepMs()); log.info("VespaStorage: Delaying startup by " + delay + " ms"); try { Thread.sleep(delay); diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index a05d2a35e4f..99928fd80fb 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -113,7 +113,7 @@ public class VespaConfiguration { } - public int randomSartupSleepMs() { + public int randomStartupSleepMs() { return getInt(RANDOM_STARTUP_SLEEP, 30000); } @@ -194,7 +194,7 @@ public class VespaConfiguration { sb.append(ROUTE + ": " + route() +"\n"); sb.append(MAX_SLEEP_TIME_MS + ": " + maxSleepTimeMs() +"\n"); sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n"); - sb.append(RANDOM_STARTUP_SLEEP + ": " + randomSartupSleepMs() +"\n"); + sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n"); sb.append(NUM_RETRIES + ": " + numRetries() +"\n"); return sb.toString(); } diff --git a/vespa_jersey2/pom.xml b/vespa_jersey2/pom.xml index bef8938fc5c..3250cd8a41f 100644 --- a/vespa_jersey2/pom.xml +++ b/vespa_jersey2/pom.xml @@ -46,6 +46,10 @@ <groupId>com.fasterxml.jackson.datatype</groupId> <artifactId>jackson-datatype-jdk8</artifactId> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jsr310</artifactId> + </dependency> </dependencies> <build> diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 8cd296b61c2..76f5d69b3a4 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -111,6 +111,7 @@ populate_environment PATH=$VESPA_HOME/bin64:$VESPA_HOME/bin:/usr/local/bin:/usr/X11R6/bin:/sbin:/bin:/usr/sbin:/usr/bin export LD_LIBRARY_PATH=$VESPA_HOME/lib64 +export MALLOC_ARENA_MAX=1 # how to find the "java" program? # should be available in $VESPA_HOME/bin or JAVA_HOME diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 89be9fec2be..fa7b9a1fe49 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -89,12 +89,11 @@ fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs fixdir ${VESPA_USER} wheel 755 logs/vespa/configserver fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs fixdir ${VESPA_USER} wheel 755 logs/vespa/search -fixdir ${VESPA_USER} wheel 755 var/cache/vespa/config fixdir ${VESPA_USER} wheel 755 var/db/vespa fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/applications +fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/tenants fixdir ${VESPA_USER} wheel 755 var/db/vespa/index fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol fixdir ${VESPA_USER} wheel 755 var/db/vespa/search @@ -102,7 +101,6 @@ fixdir ${VESPA_USER} wheel 755 var/jdisc_core fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ -fixdir ${VESPA_USER} wheel 775 libexec/vespa/modelplugins chown -hR ${VESPA_USER} logs/vespa chown -hR ${VESPA_USER} var/db/vespa diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java index 4c15b6e2365..6c5dd5e3ba5 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java @@ -45,4 +45,5 @@ public class DaemonThreadFactory implements ThreadFactory { } return t; } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 6750c99bf98..c207dabca3a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -125,8 +125,12 @@ public class IndexedTensor implements Tensor { if (indexes.length == 0) return 0; // for speed int valueIndex = 0; - for (int i = 0; i < indexes.length; i++) + for (int i = 0; i < indexes.length; i++) { + if (indexes[i] >= sizes.size(i)) { + throw new IndexOutOfBoundsException(); + } valueIndex += productOfDimensionsAfter(i, sizes) * indexes[i]; + } return valueIndex; } @@ -134,8 +138,12 @@ public class IndexedTensor implements Tensor { if (address.isEmpty()) return 0; int valueIndex = 0; - for (int i = 0; i < address.size(); i++) + for (int i = 0; i < address.size(); i++) { + if (address.intLabel(i) >= sizes.size(i)) { + throw new IndexOutOfBoundsException(); + } valueIndex += productOfDimensionsAfter(i, sizes) * address.intLabel(i); + } return valueIndex; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java new file mode 100644 index 00000000000..79bb27fcd1b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -0,0 +1,441 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A mixed tensor type. This is class is currently suitable for serialization + * and deserialization, not yet for computation. + * + * A mixed tensor has a combination of mapped and indexed dimensions. By + * reordering the mapped dimensions before the indexed dimensions, one can + * think of mixed tensors as the mapped dimensions mapping to a + * dense tensor. This dense tensor is called a dense subspace. + * + * @author lesters + */ +@Beta +public class MixedTensor implements Tensor { + + /** The dimension specification for this tensor */ + private final TensorType type; + + /** The list of cells in the tensor */ + private final ImmutableList<Cell> cells; + + /** An index structure over the cell list */ + private final Index index; + + private MixedTensor(TensorType type, ImmutableList<Cell> cells, Index index) { + this.type = type; + this.cells = ImmutableList.copyOf(cells); + this.index = index; + } + + /** Returns the tensor type */ + @Override + public TensorType type() { return type; } + + /** Returns the size of the tensor measured in number of cells */ + @Override + public int size() { return cells.size(); } + + /** Returns the value at the given address */ + @Override + public double get(TensorAddress address) { + int cellIndex = index.indexOf(address); + Cell cell = cells.get(cellIndex); + if (!address.equals(cell.getKey())) { + throw new IllegalStateException("Unable to find correct cell by direct index."); + } + return cell.getValue(); + } + + /** + * Returns an iterator over the cells of this tensor. + * Cells are returned in order of increasing indexes in the + * indexed dimensions, increasing indexes of later dimensions + * in the dimension type before earlier. No guarantee is + * given for the order of sparse dimensions. + */ + @Override + public Iterator<Cell> cellIterator() { + return cells.iterator(); + } + + /** + * Returns an iterator over the values of this tensor. + * The iteration order is the same as for cellIterator. + */ + @Override + public Iterator<Double> valueIterator() { + return new Iterator<Double>() { + Iterator<Cell> cellIterator = cellIterator(); + @Override + public boolean hasNext() { + return cellIterator.hasNext(); + } + @Override + public Double next() { + return cellIterator.next().getValue(); + } + }; + } + + @Override + public Map<TensorAddress, Double> cells() { + ImmutableMap.Builder<TensorAddress, Double> builder = new ImmutableMap.Builder<>(); + for (Cell cell : cells) { + builder.put(cell.getKey(), cell.getValue()); + } + return builder.build(); + } + + @Override + public int hashCode() { return cells.hashCode(); } + + @Override + public String toString() { return Tensor.toStandardString(this); } + + @Override + public boolean equals(Object other) { + if ( ! ( other instanceof Tensor)) return false; + return Tensor.equals(this, ((Tensor)other)); + } + + /** Returns the size of dense subspaces */ + public int denseSubspaceSize() { + return index.denseSubspaceSize(); + } + + + /** + * Base class for building mixed tensors. + */ + public abstract static class Builder implements Tensor.Builder { + + final TensorType type; + + /** + * Create a builder depending upon the type of indexed dimensions. + * If at least one indexed dimension is unbound, we create + * a temporary structure while finding dimension bounds. + */ + public static Builder of(TensorType type) { + if (type.dimensions().stream().anyMatch(d -> d instanceof TensorType.IndexedUnboundDimension)) { + return new UnboundBuilder(type); + } else { + return new BoundBuilder(type); + } + } + + private Builder(TensorType type) { + this.type = type; + } + + @Override + public TensorType type() { + return type; + } + + @Override + public Tensor.Builder cell(double value, int... labels) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public CellBuilder cell() { + return new CellBuilder(type(), this); + } + + @Override + public abstract MixedTensor build(); + + } + + + /** + * Builder for mixed tensors with bound indexed dimensions. + */ + public static class BoundBuilder extends Builder { + + /** For each sparse partial address, hold a dense subspace */ + final private Map<TensorAddress, double[]> denseSubspaceMap = new HashMap<>(); + final private Index.Builder indexBuilder; + final private Index index; + + private BoundBuilder(TensorType type) { + super(type); + indexBuilder = new Index.Builder(type); + index = indexBuilder.index(); + } + + public int denseSubspaceSize() { + return index.denseSubspaceSize(); + } + + private double[] denseSubspace(TensorAddress sparsePartial) { + if (!denseSubspaceMap.containsKey(sparsePartial)) { + denseSubspaceMap.put(sparsePartial, new double[denseSubspaceSize()]); + } + return denseSubspaceMap.get(sparsePartial); + } + + @Override + public Tensor.Builder cell(TensorAddress address, double value) { + TensorAddress sparsePart = index.sparsePartialAddress(address); + int denseOffset = index.denseOffset(address); + double[] denseSubspace = denseSubspace(sparsePart); + denseSubspace[denseOffset] = value; + return this; + } + + public Tensor.Builder block(TensorAddress sparsePart, double[] values) { + double[] denseSubspace = denseSubspace(sparsePart); + System.arraycopy(values, 0, denseSubspace, 0, denseSubspaceSize()); + return this; + } + + @Override + public MixedTensor build() { + int count = 0; + ImmutableList.Builder<Cell> builder = new ImmutableList.Builder<>(); + + for (Map.Entry<TensorAddress, double[]> entry : denseSubspaceMap.entrySet()) { + TensorAddress sparsePart = entry.getKey(); + indexBuilder.put(sparsePart, count); + + double[] denseSubspace = entry.getValue(); + for (int offset = 0; offset < denseSubspace.length; ++offset) { + TensorAddress cellAddress = index.addressOf(sparsePart, offset); + double value = denseSubspace[offset]; + builder.add(new Cell(cellAddress, value)); + count++; + } + } + return new MixedTensor(type, builder.build(), indexBuilder.build()); + } + + } + + + /** + * Temporarily stores all cells to find bounds of indexed dimensions, + * then creates a tensor using BoundBuilder. This is due to the + * fact that for serialization the size of the dense subspace must be + * known, and equal for all dense subspaces. A side effect is that the + * tensor type is effectively changed, such that unbound indexed + * dimensions become bound. + */ + public static class UnboundBuilder extends Builder { + + private Map<TensorAddress, Double> cells; + private final int[] dimensionBounds; + + private UnboundBuilder(TensorType type) { + super(type); + cells = new HashMap<>(); + dimensionBounds = new int[type.dimensions().size()]; + } + + @Override + public Tensor.Builder cell(TensorAddress address, double value) { + cells.put(address, value); + trackBounds(address); + return this; + } + + @Override + public MixedTensor build() { + TensorType boundType = createBoundType(); + BoundBuilder builder = new BoundBuilder(boundType); + for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { + builder.cell(cell.getKey(), cell.getValue()); + } + return builder.build(); + } + + public void trackBounds(TensorAddress address) { + for (int i = 0; i < type.dimensions().size(); ++i) { + TensorType.Dimension dimension = type.dimensions().get(i); + if (dimension.isIndexed()) { + dimensionBounds[i] = Math.max(address.intLabel(i), dimensionBounds[i]); + } + } + } + + public TensorType createBoundType() { + TensorType.Builder typeBuilder = new TensorType.Builder(); + for (int i = 0; i < type.dimensions().size(); ++i) { + TensorType.Dimension dimension = type.dimensions().get(i); + if (!dimension.isIndexed()) { + typeBuilder.mapped(dimension.name()); + } else { + int size = dimension.size().orElse(dimensionBounds[i] + 1); + typeBuilder.indexed(dimension.name(), size); + } + } + return typeBuilder.build(); + } + + } + + /** + * An immutable index into a list of cells. + * Contains additional information required + * for handling mixed tensor addresses. + * Assumes indexed dimensions are bound. + */ + private static class Index { + + private final TensorType type; + private final TensorType sparseType; + private final TensorType denseType; + private final List<TensorType.Dimension> mappedDimensions; + private final List<TensorType.Dimension> indexedDimensions; + + private ImmutableMap<TensorAddress, Integer> sparseMap; + private int denseSubspaceSize = -1; + + private Index(TensorType type) { + this.type = type; + this.mappedDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); + this.indexedDimensions = type.dimensions().stream().filter(d -> d.isIndexed()).collect(Collectors.toList()); + this.sparseType = createPartialType(mappedDimensions); + this.denseType = createPartialType(indexedDimensions); + } + + public int indexOf(TensorAddress address) { + TensorAddress sparsePart = sparsePartialAddress(address); + if (!sparseMap.containsKey(sparsePart)) { + throw new IllegalArgumentException("Address not found"); + } + int base = sparseMap.get(sparsePart); + int offset = denseOffset(address); + return base + offset; + } + + public static class Builder { + private final Index index; + private final ImmutableMap.Builder<TensorAddress, Integer> builder; + + public Builder(TensorType type) { + index = new Index(type); + builder = new ImmutableMap.Builder<>(); + } + + public void put(TensorAddress address, int index) { + builder.put(address, index); + } + + public Index build() { + index.sparseMap = builder.build(); + return index; + } + + public Index index() { + return index; + } + } + + public int denseSubspaceSize() { + if (denseSubspaceSize == -1) { + denseSubspaceSize = 1; + for (int i = 0; i < type.dimensions().size(); ++i) { + TensorType.Dimension dimension = type.dimensions().get(i); + if (dimension.isIndexed()) { + denseSubspaceSize *= dimension.size().orElseThrow(() -> + new IllegalArgumentException("Unknown size of indexed dimension.")); + } + } + } + return denseSubspaceSize; + } + + private TensorAddress sparsePartialAddress(TensorAddress address) { + if (type.dimensions().size() != address.size()) { + throw new IllegalArgumentException("Tensor type and address are not of same size."); + } + TensorAddress.Builder builder = new TensorAddress.Builder(sparseType); + for (int i = 0; i < type.dimensions().size(); ++i) { + TensorType.Dimension dimension = type.dimensions().get(i); + if (!dimension.isIndexed()) { + builder.add(dimension.name(), address.label(i)); + } + } + return builder.build(); + } + + private int denseOffset(TensorAddress address) { + int innerSize = 1; + int offset = 0; + for (int i = type.dimensions().size(); --i >= 0; ) { + TensorType.Dimension dimension = type.dimensions().get(i); + if (dimension.isIndexed()) { + int label = address.intLabel(i); + offset += label * innerSize; + innerSize *= dimension.size().orElseThrow(() -> + new IllegalArgumentException("Unknown size of indexed dimension.")); + } + } + return offset; + } + + private TensorAddress denseOffsetToAddress(int denseOffset) { + if (denseOffset < 0 || denseOffset > denseSubspaceSize) { + throw new IllegalArgumentException("Offset out of bounds"); + } + + int restSize = denseOffset; + int innerSize = denseSubspaceSize; + int[] labels = new int[indexedDimensions.size()]; + + for (int i = 0; i < labels.length; ++i) { + TensorType.Dimension dimension = indexedDimensions.get(i); + int dimensionSize = dimension.size().orElseThrow(() -> + new IllegalArgumentException("Unknown size of indexed dimension.")); + + innerSize /= dimensionSize; + labels[i] = restSize / innerSize; + restSize %= innerSize; + } + return TensorAddress.of(labels); + } + + private TensorAddress addressOf(TensorAddress sparsePart, int denseOffset) { + TensorAddress densePart = denseOffsetToAddress(denseOffset); + String[] labels = new String[type.dimensions().size()]; + int mappedIndex = 0; + int indexedIndex = 0; + for (TensorType.Dimension d : type.dimensions()) { + if (d.isIndexed()) { + labels[mappedIndex + indexedIndex] = densePart.label(indexedIndex); + indexedIndex++; + } else { + labels[mappedIndex + indexedIndex] = sparsePart.label(mappedIndex); + mappedIndex++; + } + } + return TensorAddress.of(labels); + } + + } + + public static TensorType createPartialType(List<TensorType.Dimension> dimensions) { + TensorType.Builder builder = new TensorType.Builder(); + for (TensorType.Dimension dimension : dimensions) { + builder.set(dimension); + } + return builder.build(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 5e3af70cba4..2ed211539d8 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -171,12 +171,16 @@ public interface Tensor { default Tensor max(Tensor argument) { return join(argument, (a, b) -> (a > b ? a : b )); } default Tensor min(Tensor argument) { return join(argument, (a, b) -> (a < b ? a : b )); } default Tensor atan2(Tensor argument) { return join(argument, Math::atan2); } + default Tensor pow(Tensor argument) { return join(argument, Math::pow); } + default Tensor fmod(Tensor argument) { return join(argument, (a, b) -> ( a % b )); } + default Tensor ldexp(Tensor argument) { return join(argument, (a, b) -> ( a * Math.pow(2.0, (int)b) )); } default Tensor larger(Tensor argument) { return join(argument, (a, b) -> ( a > b ? 1.0 : 0.0)); } default Tensor largerOrEqual(Tensor argument) { return join(argument, (a, b) -> ( a >= b ? 1.0 : 0.0)); } default Tensor smaller(Tensor argument) { return join(argument, (a, b) -> ( a < b ? 1.0 : 0.0)); } default Tensor smallerOrEqual(Tensor argument) { return join(argument, (a, b) -> ( a <= b ? 1.0 : 0.0)); } default Tensor equal(Tensor argument) { return join(argument, (a, b) -> ( a == b ? 1.0 : 0.0)); } default Tensor notEqual(Tensor argument) { return join(argument, (a, b) -> ( a != b ? 1.0 : 0.0)); } + default Tensor approxEqual(Tensor argument) { return join(argument, (a, b) -> ( approxEquals(a,b) ? 1.0 : 0.0)); } default Tensor avg(String dimension) { return avg(Collections.singletonList(dimension)); } default Tensor avg(List<String> dimensions) { return reduce(Reduce.Aggregator.avg, dimensions); } @@ -259,11 +263,27 @@ public interface Tensor { if ( a.size() != b.size()) return false; for (Iterator<Cell> aIterator = a.cellIterator(); aIterator.hasNext(); ) { Cell aCell = aIterator.next(); - if ( ! aCell.getValue().equals(b.get(aCell.getKey()))) return false; + double aValue = aCell.getValue(); + double bValue = b.get(aCell.getKey()); + if (!approxEquals(aValue, bValue, 1e-6)) return false; } return true; } + static boolean approxEquals(double x, double y, double tolerance) { + return Math.abs(x-y) < tolerance; + } + + static boolean approxEquals(double x, double y) { + if (y < -1.0 || y > 1.0) { + x = Math.nextAfter(x/y, 1.0); + y = 1.0; + } else { + x = Math.nextAfter(x, y); + } + return x==y; + } + // ----------------- Factories /** @@ -347,7 +367,7 @@ public interface Tensor { boolean containsIndexed = type.dimensions().stream().anyMatch(d -> d.isIndexed()); boolean containsMapped = type.dimensions().stream().anyMatch( d -> ! d.isIndexed()); if (containsIndexed && containsMapped) - throw new IllegalArgumentException("Combining indexed and mapped dimensions is not supported yet"); + return MixedTensor.Builder.of(type); if (containsMapped) return MappedTensor.Builder.of(type); else // indexed or empty @@ -359,7 +379,7 @@ public interface Tensor { boolean containsIndexed = type.dimensions().stream().anyMatch(d -> d.isIndexed()); boolean containsMapped = type.dimensions().stream().anyMatch( d -> ! d.isIndexed()); if (containsIndexed && containsMapped) - throw new IllegalArgumentException("Combining indexed and mapped dimensions is not supported yet"); + return MixedTensor.Builder.of(type); if (containsMapped) return MappedTensor.Builder.of(type); else // indexed or empty diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java index 401f9a10eda..1dbb94fdb20 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java @@ -134,9 +134,7 @@ public class Concat extends PrimitiveTensorFunction { if (currentDimension.equals(concatDimension)) concatSizes.set(i, aSize + bSize); else if (aSize != 0 && bSize != 0 && aSize!=bSize ) - throw new IllegalArgumentException("Dimension " + currentDimension + " must be of the same size when " + - "concatenating " + a.type() + " and " + b.type() + " along dimension " + - concatDimension + ", but was " + aSize + " and " + bSize); + concatSizes.set(i, Math.min(aSize, bSize)); else concatSizes.set(i, Math.max(aSize, bSize)); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java new file mode 100644 index 00000000000..61dfa888567 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java @@ -0,0 +1,129 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.MixedTensor; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.tensor.TensorType; + +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Implementation of a mixed binary format for a tensor. + * See eval/src/vespa/eval/tensor/serialization/format.txt for format. + * + * @author lesters + */ +@Beta +class MixedBinaryFormat implements BinaryFormat { + + @Override + public void encode(GrowableByteBuffer buffer, Tensor tensor) { + if ( ! ( tensor instanceof MixedTensor)) + throw new RuntimeException("The mixed format is only supported for mixed tensors"); + MixedTensor mixed = (MixedTensor) tensor; + encodeSparseDimensions(buffer, mixed); + encodeDenseDimensions(buffer, mixed); + encodeCells(buffer, mixed); + } + + private void encodeSparseDimensions(GrowableByteBuffer buffer, MixedTensor tensor) { + List<TensorType.Dimension> sparseDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); + buffer.putInt1_4Bytes(sparseDimensions.size()); + for (TensorType.Dimension dimension : sparseDimensions) { + buffer.putUtf8String(dimension.name()); + } + } + + private void encodeDenseDimensions(GrowableByteBuffer buffer, MixedTensor tensor) { + List<TensorType.Dimension> denseDimensions = tensor.type().dimensions().stream().filter(d -> d.isIndexed()).collect(Collectors.toList()); + buffer.putInt1_4Bytes(denseDimensions.size()); + for (TensorType.Dimension dimension : denseDimensions) { + buffer.putUtf8String(dimension.name()); + buffer.putInt1_4Bytes(dimension.size().orElseThrow(() -> + new IllegalArgumentException("Unknown size of indexed dimension."))); + } + } + + private void encodeCells(GrowableByteBuffer buffer, MixedTensor tensor) { + List<TensorType.Dimension> sparseDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); + int denseSubspaceSize = tensor.denseSubspaceSize(); + if (sparseDimensions.size() > 0) { + buffer.putInt1_4Bytes(tensor.size() / denseSubspaceSize); + } + Iterator<Tensor.Cell> cellIterator = tensor.cellIterator(); + while (cellIterator.hasNext()) { + Tensor.Cell cell = cellIterator.next(); + for (TensorType.Dimension dimension : sparseDimensions) { + int index = tensor.type().indexOfDimension(dimension.name()).orElseThrow(() -> + new IllegalStateException("Dimension not found in address.")); + buffer.putUtf8String(cell.getKey().label(index)); + } + buffer.putDouble(cell.getValue()); + for (int i = 1; i < denseSubspaceSize; ++i ) { + buffer.putDouble(cellIterator.next().getValue()); + } + } + } + + @Override + public Tensor decode(Optional<TensorType> optionalType, GrowableByteBuffer buffer) { + TensorType type; + if (optionalType.isPresent()) { + type = optionalType.get(); + TensorType serializedType = decodeType(buffer); + if ( ! serializedType.isAssignableTo(type)) + throw new IllegalArgumentException("Type/instance mismatch: A tensor of type " + serializedType + + " cannot be assigned to type " + type); + } + else { + type = decodeType(buffer); + } + MixedTensor.BoundBuilder builder = (MixedTensor.BoundBuilder)MixedTensor.Builder.of(type); + decodeCells(buffer, builder, type); + return builder.build(); + } + + private TensorType decodeType(GrowableByteBuffer buffer) { + TensorType.Builder builder = new TensorType.Builder(); + int numMappedDimensions = buffer.getInt1_4Bytes(); + for (int i = 0; i < numMappedDimensions; ++i) { + builder.mapped(buffer.getUtf8String()); + } + int numIndexedDimensions = buffer.getInt1_4Bytes(); + for (int i = 0; i < numIndexedDimensions; ++i) { + builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); + } + return builder.build(); + } + + private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) { + List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); + TensorType sparseType = MixedTensor.createPartialType(sparseDimensions); + int denseSubspaceSize = builder.denseSubspaceSize(); + + int numBlocks = 1; + if (sparseDimensions.size() > 0) { + numBlocks = buffer.getInt1_4Bytes(); + } + + double[] denseSubspace = new double[denseSubspaceSize]; + for (int i = 0; i < numBlocks; ++i) { + TensorAddress.Builder sparseAddress = new TensorAddress.Builder(sparseType); + for (TensorType.Dimension sparseDimension : sparseDimensions) { + sparseAddress.add(sparseDimension.name(), buffer.getUtf8String()); + } + for (int denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) { + denseSubspace[denseOffset] = buffer.getDouble(); + } + builder.block(sparseAddress.build(), denseSubspace); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java index 657d262b401..7467554790a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -4,6 +4,7 @@ package com.yahoo.tensor.serialization; import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -23,10 +24,15 @@ public class TypedBinaryFormat { private static final int SPARSE_BINARY_FORMAT_TYPE = 1; private static final int DENSE_BINARY_FORMAT_TYPE = 2; + private static final int MIXED_BINARY_FORMAT_TYPE = 3; public static byte[] encode(Tensor tensor) { GrowableByteBuffer buffer = new GrowableByteBuffer(); - if (tensor instanceof IndexedTensor) { + if (tensor instanceof MixedTensor) { + buffer.putInt1_4Bytes(MIXED_BINARY_FORMAT_TYPE); + new MixedBinaryFormat().encode(buffer, tensor); + } + else if (tensor instanceof IndexedTensor) { buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); new DenseBinaryFormat().encode(buffer, tensor); } @@ -51,6 +57,7 @@ public class TypedBinaryFormat { public static Tensor decode(Optional<TensorType> type, GrowableByteBuffer buffer) { int formatType = buffer.getInt1_4Bytes(); switch (formatType) { + case MIXED_BINARY_FORMAT_TYPE: return new MixedBinaryFormat().decode(type, buffer); case SPARSE_BINARY_FORMAT_TYPE: return new SparseBinaryFormat().decode(type, buffer); case DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat().decode(type, buffer); default: throw new IllegalArgumentException("Binary format type " + formatType + " is unknown"); diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java new file mode 100644 index 00000000000..fef8f05f4e1 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java @@ -0,0 +1,155 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.tensor; + +import com.google.common.collect.Sets; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Basic mixed tensor tests. Tensor operations are tested in EvaluationTestCase + * + * @author lesters + */ +public class MixedTensorTestCase { + + @Test + public void testEmpty() { + TensorType type = new TensorType.Builder().mapped("x").indexed("y", 3).build(); + Tensor empty = Tensor.Builder.of(type).build(); + assertTrue(empty instanceof MixedTensor); + assertTrue(empty.isEmpty()); + assertEquals("tensor(x{},y[3]):{}", empty.toString()); + assertEquals("tensor(x{},y[3]):{}", Tensor.from("tensor(x{},y[3]):{}").toString()); + } + + @Test + public void testScalar() { + TensorType type = new TensorType.Builder().build(); + Tensor scalar = MixedTensor.Builder.of(type).cell().value(42.0).build(); + assertEquals(scalar.asDouble(), 42.0, 1e-6); + } + + @Test + public void testOneIndexedBuilding() { + TensorType type = new TensorType.Builder().indexed("y", 3).build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("y", 0).value(1). + cell().label("y", 1).value(2). + // {y:2} should be 0.0 and non NaN since we specify indexed size + build(); + assertEquals(Sets.newHashSet("y"), tensor.type().dimensionNames()); + assertEquals("{{y:0}:1.0,{y:1}:2.0,{y:2}:0.0}", + tensor.toString()); + } + + @Test + public void testTwoIndexedBuilding() { + TensorType type = new TensorType.Builder().indexed("x").indexed("y", 3).build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", 0).label("y", 0).value(1). + cell().label("x", 0).label("y", 1).value(2). + // {x:1,y:2} should be 0.0 and non NaN since we specify indexed size + cell().label("x", 1).label("y", 0).value(4). + cell().label("x", 1).label("y", 1).value(5). + cell().label("x", 1).label("y", 2).value(6). + build(); + assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); + assertEquals("{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:0,y:2}:0.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", + tensor.toString()); + } + + @Test + public void testOneMappedBuilding() { + TensorType type = new TensorType.Builder().mapped("x").build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", "0").value(1). + cell().label("x", "1").value(2). + build(); + assertEquals(Sets.newHashSet("x"), tensor.type().dimensionNames()); + assertEquals("{{x:0}:1.0,{x:1}:2.0}", + tensor.toString()); + } + + @Test + public void testTwoMappedBuilding() { + TensorType type = new TensorType.Builder().mapped("x").mapped("y").build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", "0").label("y", "0").value(1). + cell().label("x", "0").label("y", "1").value(2). + cell().label("x", "1").label("y", "0").value(4). + cell().label("x", "1").label("y", "1").value(5). + cell().label("x", "1").label("y", "2").value(6). + build(); + assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); + assertEquals("{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", + tensor.toString()); + } + + @Test + public void testOneMappedOneIndexedBuilding() { + TensorType type = new TensorType.Builder().mapped("x").indexed("y", 3).build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", "1").label("y", 0).value(1). + cell().label("x", "1").label("y", 1).value(2). + // {x:1,y:2} should be 0.0 and non NaN since we specify indexed size + cell().label("x", "2").label("y", 0).value(4). + cell().label("x", "2").label("y", 1).value(5). + cell().label("x", "2").label("y", 2).value(6). + build(); + assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); + assertEquals("{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}", + tensor.toString()); + } + + @Test + public void testTwoMappedOneIndexedBuilding() { + TensorType type = new TensorType.Builder().mapped("x").indexed("y").mapped("z").build(); + Tensor tensor = Tensor.Builder.of(type). + cell().label("x", "x1").label("y", 0).label("z","z1").value(1). + cell().label("x", "x1").label("y", 0).label("z","z2").value(2). + cell().label("x", "x1").label("y", 1).label("z","z1").value(3). + cell().label("x", "x1").label("y", 1).label("z","z2").value(4). + cell().label("x", "x1").label("y", 2).label("z","z1").value(5). + cell().label("x", "x1").label("y", 2).label("z","z2").value(6). + cell().label("x", "x2").label("y", 0).label("z","z1").value(11). + cell().label("x", "x2").label("y", 0).label("z","z2").value(12). + cell().label("x", "x2").label("y", 1).label("z","z1").value(13). + cell().label("x", "x2").label("y", 1).label("z","z2").value(14). + cell().label("x", "x2").label("y", 2).label("z","z1").value(15). + cell().label("x", "x2").label("y", 2).label("z","z2").value(16). + build(); + assertEquals(Sets.newHashSet("x", "y", "z"), tensor.type().dimensionNames()); + assertEquals("{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}", + tensor.toString()); + } + + @Test + public void testTwoMappedTwoIndexedBuilding() { + TensorType type = new TensorType.Builder().mapped("i").indexed("j", 2).mapped("k").indexed("l", 2).build(); + Tensor tensor = Tensor.Builder.of(type). + cell().label("i", "a").label("k","c").label("j",0).label("l",0).value(1). + cell().label("i", "a").label("k","c").label("j",0).label("l",1).value(2). + cell().label("i", "a").label("k","c").label("j",1).label("l",0).value(3). + cell().label("i", "a").label("k","c").label("j",1).label("l",1).value(4). + cell().label("i", "a").label("k","d").label("j",0).label("l",0).value(5). + cell().label("i", "a").label("k","d").label("j",0).label("l",1).value(6). + cell().label("i", "a").label("k","d").label("j",1).label("l",0).value(7). + cell().label("i", "a").label("k","d").label("j",1).label("l",1).value(8). + cell().label("i", "b").label("k","c").label("j",0).label("l",0).value(9). + cell().label("i", "b").label("k","c").label("j",0).label("l",1).value(10). + cell().label("i", "b").label("k","c").label("j",1).label("l",0).value(11). + cell().label("i", "b").label("k","c").label("j",1).label("l",1).value(12). + cell().label("i", "b").label("k","d").label("j",0).label("l",0).value(13). + cell().label("i", "b").label("k","d").label("j",0).label("l",1).value(14). + cell().label("i", "b").label("k","d").label("j",1).label("l",0).value(15). + cell().label("i", "b").label("k","d").label("j",1).label("l",1).value(16). + build(); + assertEquals(Sets.newHashSet("i", "j", "k", "l"), tensor.type().dimensionNames()); + assertEquals("{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}", + tensor.toString()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java index a653ef97734..7e1f292eb7b 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java @@ -43,12 +43,7 @@ public class ConcatTestCase { Tensor a = Tensor.from("tensor(x[]):{ {x:0}:1, {x:1}:2 }"); Tensor b = Tensor.from("tensor(x[]):{ {x:0}:4, {x:1}:5, {x:2}:6 }"); assertEquals(Tensor.from("tensor(x[5]):{ {x:0}:1, {x:1}:2, {x:2}:4, {x:3}:5, {x:4}:6 }"), a.concat(b, "x")); - try { - a.concat(b, "y"); - fail("Expected exception"); - } catch (IllegalArgumentException expected) { - // success - } + assertEquals(Tensor.from("tensor(x[2],y[2]):{ {x:0,y:0}:1, {x:1,y:0}:2, {x:0,y:1}:4, {x:1,y:1}:5 }"), a.concat(b, "y")); } @Test diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java new file mode 100644 index 00000000000..b1d7d797b3e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java @@ -0,0 +1,95 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.tensor.serialization; + +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.MixedTensor; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * Tests for the mixed binary format. + * + * @author lesters + */ +public class MixedBinaryFormatTestCase { + + @Test + public void testSerialization() { + assertSerialization("tensor(x{},y[3]):{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}"); + assertSerialization("tensor(x{},y[]):{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}"); + + assertSerialization("tensor(x{},y[3],z{}):{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}"); + assertSerialization("tensor(x{},y[],z{}):{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}"); + + assertSerialization("tensor(i{},j[2],k{},l[2]):{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}"); + assertSerialization("tensor(i{},j[],k{},l[]):{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}"); + } + + @Test + public void testOneIndexedSerialization() { + TensorType type = new TensorType.Builder().indexed("y", 3).build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("y", 0).value(1). + cell().label("y", 1).value(2). + build(); + assertSerialization(tensor); + } + + @Test + public void testTwoIndexedSerialization() { + TensorType type = new TensorType.Builder().indexed("x").indexed("y", 3).build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", 0).label("y", 0).value(1). + cell().label("x", 0).label("y", 1).value(2). + cell().label("x", 1).label("y", 0).value(4). + cell().label("x", 1).label("y", 1).value(5). + cell().label("x", 1).label("y", 2).value(6). + build(); + assertSerialization(tensor); + } + + @Test + public void testOneMappedSerialization() { + TensorType type = new TensorType.Builder().mapped("x").build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", "0").value(1). + cell().label("x", "1").value(2). + build(); + assertSerialization(tensor); + } + + @Test + public void testTwoMappedSerialization() { + TensorType type = new TensorType.Builder().mapped("x").mapped("y").build(); + Tensor tensor = MixedTensor.Builder.of(type). + cell().label("x", "0").label("y", "0").value(1). + cell().label("x", "0").label("y", "1").value(2). + cell().label("x", "1").label("y", "0").value(4). + cell().label("x", "1").label("y", "1").value(5). + cell().label("x", "1").label("y", "2").value(6). + build(); + assertSerialization(tensor); + } + + private void assertSerialization(String tensorString) { + assertSerialization(Tensor.from(tensorString)); + } + + private void assertSerialization(Tensor tensor) { + assertSerialization(tensor, tensor.type()); + } + + private void assertSerialization(Tensor tensor, TensorType expectedType) { + byte[] encodedTensor = TypedBinaryFormat.encode(tensor); + Tensor decodedTensor = TypedBinaryFormat.decode(Optional.of(expectedType), GrowableByteBuffer.wrap(encodedTensor)); + assertEquals(tensor, decodedTensor); + } + +} + diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SerializationTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SerializationTestCase.java new file mode 100644 index 00000000000..68bf59e3ed9 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SerializationTestCase.java @@ -0,0 +1,149 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.tensor.serialization; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SerializationTestCase { + + private static String testPath = "eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json"; + private static List<String> tests = new ArrayList<>(); + + @Before + public void loadTests() throws IOException { + File testSpec = new File(testPath); + if (!testSpec.exists()) { + testSpec = new File("../" + testPath); + } + try(BufferedReader br = new BufferedReader(new FileReader(testSpec))) { + String test = br.readLine(); + while (test != null) { + tests.add(test); + test = br.readLine(); + } + } + } + + @Test + public void testSerialization() throws IOException { + for (String test : tests) { + ObjectMapper mapper = new ObjectMapper(); + JsonNode node = mapper.readTree(test); + if (node.has("tensor") && node.has("binary")) { + System.out.println("Running test: " + test); + + Tensor tensor = buildTensor(node.get("tensor")); + String spec = getSpec(node.get("tensor")); + byte[] encodedTensor = TypedBinaryFormat.encode(tensor); + boolean serializedToABinaryRepresentation = false; + + JsonNode binaryNode = node.get("binary"); + for (int i = 0; i < binaryNode.size(); ++i) { + byte[] bin = getBytes(binaryNode.get(i).asText()); + Tensor decodedTensor = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(bin)); + + if (spec.equalsIgnoreCase("double")) { + assertEquals(tensor.asDouble(), decodedTensor.asDouble(), 1e-6); + } else { + assertEquals(tensor, decodedTensor); + } + + if (Arrays.equals(encodedTensor, bin)) { + serializedToABinaryRepresentation = true; + } + } + assertTrue("Tensor did not serialize to one of the given representations", serializedToABinaryRepresentation); + } + } + } + + private Tensor buildTensor(JsonNode tensor) { + TensorType type = tensorType(tensor); + Tensor.Builder builder = Tensor.Builder.of(type); + tensorCells(tensor, builder); + return builder.build(); + } + + private TensorType tensorType(JsonNode tensor) { + String spec = getSpec(tensor); + if (spec.equalsIgnoreCase("double")) { + spec = "tensor()"; + } + return TensorType.fromSpec(spec); + } + + private String getSpec(JsonNode tensor) { + return tensor.get("type").asText(); + } + + private void tensorCells(JsonNode tensor, Tensor.Builder builder) { + JsonNode cells = tensor.get("cells"); + for (JsonNode cell : cells) { + tensorCell(cell, builder.cell()); + } + } + + private void tensorCell(JsonNode cell, Tensor.Builder.CellBuilder cellBuilder) { + tensorCellAddress(cellBuilder, cell.get("address")); + tensorCellValue(cellBuilder, cell.get("value")); + } + + private void tensorCellValue(Tensor.Builder.CellBuilder cellBuilder, JsonNode value) { + cellBuilder.value(value.doubleValue()); + } + + private void tensorCellAddress(Tensor.Builder.CellBuilder cellBuilder, JsonNode address) { + Iterator<String> dimension = address.fieldNames(); + while (dimension.hasNext()) { + String name = dimension.next(); + JsonNode label = address.get(name); + cellBuilder.label(name, label.asText()); + } + } + + private byte[] getBytes(String binaryRepresentation) { + return parseHexValue(binaryRepresentation.substring(2)); + } + + private byte[] parseHexValue(String s) { + final int len = s.length(); + byte[] bytes = new byte[len/2]; + for (int i = 0; i < len; i += 2) { + int c1 = hexValue(s.charAt(i)) << 4; + int c2 = hexValue(s.charAt(i + 1)); + bytes[i/2] = (byte)(c1 + c2); + } + return bytes; + } + + private int hexValue(Character c) { + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } else if (c >= '0' && c <= '9') { + return c - '0'; + } + throw new IllegalArgumentException("Hex contains illegal characters"); + } + +} |